enhance: greatly reduce the loading memory overhead - by up to 25% (#43533)

issue: #43088
issue: #43038

The current loading process:

* When loading an index, we first download the index files into a list
of buffers, say A
* then constructing(copying) them into a vector of FieldDatas(each file
is a FieldData), say B
* assembles them together as a huge BinarySet, say C
* lastly, copy into the actual index data structure, say D

The problem:

* We can see that, after each step, we don't need the data in previous
step.
* But currently, we release the memory of A, B, C only after we have
finished constructing D
* This leads to a up to 4x peak memory usage comparing with the raw
index size, during the loading process
* This PR allows timely releasing of B after we assembled C. So after
this PR, the peak memory usage during loading will be up to 3x of the
raw index size.

I will create another PR to release A after we created B, that seems
more complicated and need more work.

Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>

internal/core/src/index/BitmapIndex.cpp

@@ -572,6 +572,8 @@ BitmapIndex<T>::Load(milvus::tracer::TraceContext ctx, const Config& config) {
         index_files.value(), config[milvus::LOAD_PRIORITY]);
     BinarySet binary_set;
     AssembleIndexDatas(index_datas, binary_set);
+    // clear index_datas to free memory early
+    index_datas.clear();
     LoadWithoutAssemble(binary_set, config);
 }
 

internal/core/src/index/HybridScalarIndex.cpp

@@ -369,6 +369,8 @@ HybridScalarIndex<T>::Load(milvus::tracer::TraceContext ctx,
         config[milvus::LOAD_PRIORITY]);
     BinarySet binary_set;
     AssembleIndexDatas(index_datas, binary_set);
+    // clear index_datas to free memory early
+    index_datas.clear();
     DeserializeIndexType(binary_set);
 
     auto index = GetInternalIndex();

internal/core/src/index/NgramInvertedIndex.cpp

@@ -80,6 +80,8 @@ NgramInvertedIndex::Load(milvus::tracer::TraceContext ctx,
             file, config[milvus::LOAD_PRIORITY]);
         BinarySet binary_set;
         AssembleIndexDatas(index_datas, binary_set);
+        // clear index_datas to free memory early
+        index_datas.clear();
         auto index_valid_data = binary_set.GetByName("index_null_offset");
         null_offset_.resize((size_t)index_valid_data->size / sizeof(size_t));
         memcpy(null_offset_.data(),

internal/core/src/index/ScalarIndexSort.cpp

@@ -210,6 +210,8 @@ ScalarIndexSort<T>::Load(milvus::tracer::TraceContext ctx,
         index_files.value(), config[milvus::LOAD_PRIORITY]);
     BinarySet binary_set;
     AssembleIndexDatas(index_datas, binary_set);
+    // clear index_datas to free memory early
+    index_datas.clear();
     LoadWithoutAssemble(binary_set, config);
 }
 

internal/core/src/index/StringIndexMarisa.cpp

@@ -230,6 +230,8 @@ StringIndexMarisa::Load(milvus::tracer::TraceContext ctx,
         index_files.value(), config[milvus::LOAD_PRIORITY]);
     BinarySet binary_set;
     AssembleIndexDatas(index_datas, binary_set);
+    // clear index_datas to free memory early
+    index_datas.clear();
     LoadWithoutAssemble(binary_set, config);
 }
 

internal/core/src/index/TextMatchIndex.cpp

@@ -152,6 +152,8 @@ TextMatchIndex::Load(const Config& config) {
             file, config[milvus::LOAD_PRIORITY]);
         BinarySet binary_set;
         AssembleIndexDatas(index_datas, binary_set);
+        // clear index_datas to free memory early
+        index_datas.clear();
         auto index_valid_data = binary_set.GetByName("index_null_offset");
         null_offset_.resize((size_t)index_valid_data->size / sizeof(size_t));
         memcpy(null_offset_.data(),

internal/core/src/index/VectorMemIndex.cpp

@@ -272,6 +272,8 @@ VectorMemIndex<T>::Load(milvus::tracer::TraceContext ctx,
     LOG_INFO("construct binary set...");
     BinarySet binary_set;
     AssembleIndexDatas(index_data_codecs, binary_set);
+    // clear index_data_codecs to free memory early
+    index_data_codecs.clear();
 
     // start engine load index span
     auto span_load_engine =