admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: set mmap_file_raii_ to nullptr when mmap is disabled (#44516)

Browse Source 
issue: #44510
related: #44501

Signed-off-by: Shawn Wang <shawn.wang@zilliz.com>
This commit is contained in:
sparknack 2025-09-24 11:50:03 +08:00 committed by GitHub
parent bc170201dd
commit 14c085374e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 27 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
internal/core/src/common/ChunkWriter.cpp
View File

@ -90,7 +90,9 @@ StringChunkWriter::finish() {
char padding[MMAP_STRING_PADDING]; char padding[MMAP_STRING_PADDING];
target_->write(padding, MMAP_STRING_PADDING); target_->write(padding, MMAP_STRING_PADDING);
auto [data, size] = target_->get(); auto [data, size] = target_->get();
auto mmap_file_raii = std::make_unique<MmapFileRAII>(file_path_); auto mmap_file_raii = file_path_.empty()
? nullptr
: std::make_unique<MmapFileRAII>(file_path_);
return std::make_unique<StringChunk>( return std::make_unique<StringChunk>(
row_nums_, data, size, nullable_, std::move(mmap_file_raii)); row_nums_, data, size, nullable_, std::move(mmap_file_raii));
} }
@ -153,7 +155,9 @@ JSONChunkWriter::finish() {
target_->write(padding, simdjson::SIMDJSON_PADDING); target_->write(padding, simdjson::SIMDJSON_PADDING);
auto [data, size] = target_->get(); auto [data, size] = target_->get();
auto mmap_file_raii = std::make_unique<MmapFileRAII>(file_path_); auto mmap_file_raii = file_path_.empty()
? nullptr
: std::make_unique<MmapFileRAII>(file_path_);
return std::make_unique<JSONChunk>( return std::make_unique<JSONChunk>(
row_nums_, data, size, nullable_, std::move(mmap_file_raii)); row_nums_, data, size, nullable_, std::move(mmap_file_raii));
} }
@ -241,7 +245,9 @@ ArrayChunkWriter::finish() {
char padding[MMAP_ARRAY_PADDING]; char padding[MMAP_ARRAY_PADDING];
target_->write(padding, MMAP_ARRAY_PADDING); target_->write(padding, MMAP_ARRAY_PADDING);
auto [data, size] = target_->get(); auto [data, size] = target_->get();
auto mmap_file_raii = std::make_unique<MmapFileRAII>(file_path_); auto mmap_file_raii = file_path_.empty()
? nullptr
: std::make_unique<MmapFileRAII>(file_path_);
return std::make_unique<ArrayChunk>(row_nums_, return std::make_unique<ArrayChunk>(row_nums_,
data, data,
size, size,
@ -387,7 +393,9 @@ VectorArrayChunkWriter::finish() {
target_->write(padding, MMAP_ARRAY_PADDING); target_->write(padding, MMAP_ARRAY_PADDING);
auto [data, size] = target_->get(); auto [data, size] = target_->get();
auto mmap_file_raii = std::make_unique<MmapFileRAII>(file_path_); auto mmap_file_raii = file_path_.empty()
? nullptr
: std::make_unique<MmapFileRAII>(file_path_);
return std::make_unique<VectorArrayChunk>( return std::make_unique<VectorArrayChunk>(
dim_, row_nums_, data, size, element_type_, std::move(mmap_file_raii)); dim_, row_nums_, data, size, element_type_, std::move(mmap_file_raii));
} }
@ -449,7 +457,9 @@ SparseFloatVectorChunkWriter::write(const arrow::ArrayVector& array_vec) {
std::unique_ptr<Chunk> std::unique_ptr<Chunk>
SparseFloatVectorChunkWriter::finish() { SparseFloatVectorChunkWriter::finish() {
auto [data, size] = target_->get(); auto [data, size] = target_->get();
auto mmap_file_raii = std::make_unique<MmapFileRAII>(file_path_); auto mmap_file_raii = file_path_.empty()
? nullptr
: std::make_unique<MmapFileRAII>(file_path_);
return std::make_unique<SparseFloatVectorChunk>( return std::make_unique<SparseFloatVectorChunk>(
row_nums_, data, size, nullable_, std::move(mmap_file_raii)); row_nums_, data, size, nullable_, std::move(mmap_file_raii));
} }

4
internal/core/src/common/ChunkWriter.h
View File

@ -146,7 +146,9 @@ class ChunkWriter final : public ChunkWriterBase {
std::unique_ptr<Chunk> std::unique_ptr<Chunk>
finish() override { finish() override {
auto [data, size] = target_->get(); auto [data, size] = target_->get();
auto mmap_file_raii = std::make_unique<MmapFileRAII>(file_path_); auto mmap_file_raii = file_path_.empty()
? nullptr
: std::make_unique<MmapFileRAII>(file_path_);
return std::make_unique<FixedWidthChunk>(row_nums_, return std::make_unique<FixedWidthChunk>(row_nums_,
dim_, dim_,
data, data,

2
internal/core/src/index/IndexFactory.cpp
View File

@ -305,7 +305,7 @@ IndexFactory::ScalarIndexLoadResource(
request.final_memory_cost = index_size_in_bytes; request.final_memory_cost = index_size_in_bytes;
request.final_disk_cost = 0; request.final_disk_cost = 0;
request.max_memory_cost = 2 * index_size_in_bytes; request.max_memory_cost = 2 * index_size_in_bytes;
request.max_disk_cost = 0; request.max_disk_cost = index_size_in_bytes;
} }
request.has_raw_data = true; request.has_raw_data = true;
} else if (index_type == milvus::index::INVERTED_INDEX_TYPE || } else if (index_type == milvus::index::INVERTED_INDEX_TYPE ||

8
internal/core/src/index/StringIndexMarisa.cpp
View File

@ -186,6 +186,7 @@ StringIndexMarisa::LoadWithoutAssemble(const BinarySet& set,
const Config& config) { const Config& config) {
auto uuid = boost::uuids::random_generator()(); auto uuid = boost::uuids::random_generator()();
auto uuid_string = boost::uuids::to_string(uuid); auto uuid_string = boost::uuids::to_string(uuid);
// TODO: change the mmap path of marisa index
auto file_name = std::string("/tmp/") + uuid_string; auto file_name = std::string("/tmp/") + uuid_string;
auto index = set.GetByName(MARISA_TRIE_INDEX); auto index = set.GetByName(MARISA_TRIE_INDEX);
@ -204,11 +205,16 @@ StringIndexMarisa::LoadWithoutAssemble(const BinarySet& set,
if (config.contains(MMAP_FILE_PATH)) { if (config.contains(MMAP_FILE_PATH)) {
trie_.mmap(file_name.c_str()); trie_.mmap(file_name.c_str());
mmap_file_raii_ = std::make_unique<MmapFileRAII>(file_name);
} else { } else {
auto file = File::Open(file_name, O_RDONLY); auto file = File::Open(file_name, O_RDONLY);
trie_.read(file.Descriptor()); trie_.read(file.Descriptor());
mmap_file_raii_ = nullptr;
}
if (!config.contains(MMAP_FILE_PATH)) {
unlink(file_name.c_str());
} }
mmap_file_raii_ = std::make_unique<MmapFileRAII>(file_name);
auto str_ids = set.GetByName(MARISA_STR_IDS); auto str_ids = set.GetByName(MARISA_STR_IDS);
auto str_ids_len = str_ids->size; auto str_ids_len = str_ids->size;

2
internal/core/unittest/test_loading.cpp
View File

@ -208,7 +208,7 @@ INSTANTIATE_TEST_SUITE_P(
{"mmap", "false"}, {"mmap", "false"},
{"field_type", "string"}}, {"field_type", "string"}},
{2UL * 1024 * 1024 * 1024, {2UL * 1024 * 1024 * 1024,
0UL, 1UL * 1024 * 1024 * 1024,
1UL * 1024 * 1024 * 1024, 1UL * 1024 * 1024 * 1024,
0UL, 0UL,
true}), true}),