From d367770649caf3fa1784a6740cf40ea335d83dce Mon Sep 17 00:00:00 2001 From: Buqian Zheng Date: Thu, 24 Jul 2025 11:26:54 +0800 Subject: [PATCH] enhance: greatly reduce the loading memory overhead - by up to 25% (#43533) issue: #43088 issue: #43038 The current loading process: * When loading an index, we first download the index files into a list of buffers, say A * then constructing(copying) them into a vector of FieldDatas(each file is a FieldData), say B * assembles them together as a huge BinarySet, say C * lastly, copy into the actual index data structure, say D The problem: * We can see that, after each step, we don't need the data in previous step. * But currently, we release the memory of A, B, C only after we have finished constructing D * This leads to a up to 4x peak memory usage comparing with the raw index size, during the loading process * This PR allows timely releasing of B after we assembled C. So after this PR, the peak memory usage during loading will be up to 3x of the raw index size. I will create another PR to release A after we created B, that seems more complicated and need more work. Signed-off-by: Buqian Zheng --- internal/core/src/index/BitmapIndex.cpp | 2 ++ internal/core/src/index/HybridScalarIndex.cpp | 2 ++ internal/core/src/index/NgramInvertedIndex.cpp | 2 ++ internal/core/src/index/ScalarIndexSort.cpp | 2 ++ internal/core/src/index/StringIndexMarisa.cpp | 2 ++ internal/core/src/index/TextMatchIndex.cpp | 2 ++ internal/core/src/index/VectorMemIndex.cpp | 2 ++ 7 files changed, 14 insertions(+) diff --git a/internal/core/src/index/BitmapIndex.cpp b/internal/core/src/index/BitmapIndex.cpp index db7ea750b2..0211ce3fbf 100644 --- a/internal/core/src/index/BitmapIndex.cpp +++ b/internal/core/src/index/BitmapIndex.cpp @@ -572,6 +572,8 @@ BitmapIndex::Load(milvus::tracer::TraceContext ctx, const Config& config) { index_files.value(), config[milvus::LOAD_PRIORITY]); BinarySet binary_set; AssembleIndexDatas(index_datas, binary_set); + // clear index_datas to free memory early + index_datas.clear(); LoadWithoutAssemble(binary_set, config); } diff --git a/internal/core/src/index/HybridScalarIndex.cpp b/internal/core/src/index/HybridScalarIndex.cpp index 64aa52df65..bcc7f13cbd 100644 --- a/internal/core/src/index/HybridScalarIndex.cpp +++ b/internal/core/src/index/HybridScalarIndex.cpp @@ -369,6 +369,8 @@ HybridScalarIndex::Load(milvus::tracer::TraceContext ctx, config[milvus::LOAD_PRIORITY]); BinarySet binary_set; AssembleIndexDatas(index_datas, binary_set); + // clear index_datas to free memory early + index_datas.clear(); DeserializeIndexType(binary_set); auto index = GetInternalIndex(); diff --git a/internal/core/src/index/NgramInvertedIndex.cpp b/internal/core/src/index/NgramInvertedIndex.cpp index ee3f9c7e09..e1402827fd 100644 --- a/internal/core/src/index/NgramInvertedIndex.cpp +++ b/internal/core/src/index/NgramInvertedIndex.cpp @@ -80,6 +80,8 @@ NgramInvertedIndex::Load(milvus::tracer::TraceContext ctx, file, config[milvus::LOAD_PRIORITY]); BinarySet binary_set; AssembleIndexDatas(index_datas, binary_set); + // clear index_datas to free memory early + index_datas.clear(); auto index_valid_data = binary_set.GetByName("index_null_offset"); null_offset_.resize((size_t)index_valid_data->size / sizeof(size_t)); memcpy(null_offset_.data(), diff --git a/internal/core/src/index/ScalarIndexSort.cpp b/internal/core/src/index/ScalarIndexSort.cpp index 54c519b8df..d61e7d1a93 100644 --- a/internal/core/src/index/ScalarIndexSort.cpp +++ b/internal/core/src/index/ScalarIndexSort.cpp @@ -210,6 +210,8 @@ ScalarIndexSort::Load(milvus::tracer::TraceContext ctx, index_files.value(), config[milvus::LOAD_PRIORITY]); BinarySet binary_set; AssembleIndexDatas(index_datas, binary_set); + // clear index_datas to free memory early + index_datas.clear(); LoadWithoutAssemble(binary_set, config); } diff --git a/internal/core/src/index/StringIndexMarisa.cpp b/internal/core/src/index/StringIndexMarisa.cpp index 681fb95ac7..918e6c7d78 100644 --- a/internal/core/src/index/StringIndexMarisa.cpp +++ b/internal/core/src/index/StringIndexMarisa.cpp @@ -230,6 +230,8 @@ StringIndexMarisa::Load(milvus::tracer::TraceContext ctx, index_files.value(), config[milvus::LOAD_PRIORITY]); BinarySet binary_set; AssembleIndexDatas(index_datas, binary_set); + // clear index_datas to free memory early + index_datas.clear(); LoadWithoutAssemble(binary_set, config); } diff --git a/internal/core/src/index/TextMatchIndex.cpp b/internal/core/src/index/TextMatchIndex.cpp index 9069ac2094..bbc7f84f4e 100644 --- a/internal/core/src/index/TextMatchIndex.cpp +++ b/internal/core/src/index/TextMatchIndex.cpp @@ -152,6 +152,8 @@ TextMatchIndex::Load(const Config& config) { file, config[milvus::LOAD_PRIORITY]); BinarySet binary_set; AssembleIndexDatas(index_datas, binary_set); + // clear index_datas to free memory early + index_datas.clear(); auto index_valid_data = binary_set.GetByName("index_null_offset"); null_offset_.resize((size_t)index_valid_data->size / sizeof(size_t)); memcpy(null_offset_.data(), diff --git a/internal/core/src/index/VectorMemIndex.cpp b/internal/core/src/index/VectorMemIndex.cpp index 2aef1ed92c..d4adde12bc 100644 --- a/internal/core/src/index/VectorMemIndex.cpp +++ b/internal/core/src/index/VectorMemIndex.cpp @@ -272,6 +272,8 @@ VectorMemIndex::Load(milvus::tracer::TraceContext ctx, LOG_INFO("construct binary set..."); BinarySet binary_set; AssembleIndexDatas(index_data_codecs, binary_set); + // clear index_data_codecs to free memory early + index_data_codecs.clear(); // start engine load index span auto span_load_engine =