admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: fix bug that set incorrect info to columnbase (#34428)

Browse Source 
#34427

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>
This commit is contained in:
zhagnlu 2024-07-14 22:27:46 +08:00 committed by GitHub
parent 18c83c6466
commit bd9727a1f7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 20 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
internal/core/src/common/FieldMeta.h
View File

@ -136,6 +136,7 @@ class FieldMeta {
if (is_vector()) {
return GetDataTypeSize(type_, get_dim());
} else if (is_string()) {
Assert(string_info_.has_value());
return string_info_->max_length;
} else if (IsVariableDataType(type_)) {
return type_ == DataType::ARRAY ? ARRAY_SIZE : JSON_SIZE;

38
internal/core/src/mmap/Column.h
View File

@ -64,16 +64,16 @@ class ColumnBase {
};
// memory mode ctor
ColumnBase(size_t reserve, const FieldMeta& field_meta)
: type_size_(IsSparseFloatVectorDataType(field_meta.get_data_type())
? 1
: field_meta.get_sizeof()),
mapping_type_(MappingType::MAP_WITH_ANONYMOUS) {
SetPaddingSize(field_meta.get_data_type());
: mapping_type_(MappingType::MAP_WITH_ANONYMOUS) {
auto data_type = field_meta.get_data_type();
SetPaddingSize(data_type);
if (IsVariableDataType(field_meta.get_data_type())) {
if (IsVariableDataType(data_type)) {
return;
}
type_size_ = field_meta.get_sizeof();
cap_size_ = type_size_ * reserve;
// use anon mapping so we are able to free these memory with munmap only
@ -120,12 +120,13 @@ class ColumnBase {
// mmap mode ctor
// User must call Seal to build the view for variable length column.
ColumnBase(const File& file, size_t size, const FieldMeta& field_meta)
: type_size_(IsSparseFloatVectorDataType(field_meta.get_data_type())
? 1
: field_meta.get_sizeof()),
mapping_type_(MappingType::MAP_WITH_FILE),
num_rows_(size / type_size_) {
SetPaddingSize(field_meta.get_data_type());
: mapping_type_(MappingType::MAP_WITH_FILE) {
auto data_type = field_meta.get_data_type();
SetPaddingSize(data_type);
if (!IsVariableDataType(data_type)) {
type_size_ = field_meta.get_sizeof();
num_rows_ = size / type_size_;
}
size_ = size;
cap_size_ = size;
@ -148,12 +149,7 @@ class ColumnBase {
size_t size,
int dim,
const DataType& data_type)
: type_size_(IsSparseFloatVectorDataType(data_type)
? 1
: GetDataTypeSize(data_type, dim)),
num_rows_(
IsSparseFloatVectorDataType(data_type) ? 1 : (size / type_size_)),
size_(size),
: size_(size),
cap_size_(size),
mapping_type_(MappingType::MAP_WITH_FILE) {
SetPaddingSize(data_type);
@ -161,6 +157,10 @@ class ColumnBase {
// use exact same size of file, padding shall be written in file already
// see also https://github.com/milvus-io/milvus/issues/34442
size_t mapped_size = cap_size_;
if (!IsVariableDataType(data_type)) {
type_size_ = GetDataTypeSize(data_type, dim);
num_rows_ = size / type_size_;
}
data_ = static_cast<char*>(mmap(
nullptr, mapped_size, PROT_READ, MAP_SHARED, file.Descriptor(), 0));
AssertInfo(data_ != MAP_FAILED,
@ -355,7 +355,7 @@ class ColumnBase {
size_t cap_size_{0};
size_t padding_{0};
// type_size_ is not used for sparse float vector column.
const size_t type_size_{1};
size_t type_size_{1};
size_t num_rows_{0};
// length in bytes