enhance: [2.6] add vector reserve to improve memory allocation in segcore (#45759)

This commit optimizes std::vector usage across segcore by adding
reserve() calls where the size is known in advance, reducing memory
reallocations during push_back operations.

Changes:

TimestampIndex.cpp: Reserve space for prefix_sums and timestamp_barriers
SegmentGrowingImpl.cpp: Reserve space for binlog info vectors
ChunkedSegmentSealedImpl.cpp: Reserve space for futures and field data
vectors
storagev2translator/GroupChunkTranslator.cpp: Reserve space for metadata
vectors
This improves performance by avoiding multiple memory reallocations when
the vector size is predictable.

issue: https://github.com/milvus-io/milvus/issues/45679
pr: https://github.com/milvus-io/milvus/pull/45757

Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>
This commit is contained in:
Buqian Zheng 2025-11-26 10:33:09 +08:00 committed by GitHub
parent feffbf90d2
commit ce302c1555
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 15 additions and 0 deletions

View File

@ -285,6 +285,7 @@ ChunkedSegmentSealedImpl::load_column_group_data_internal(
// warmup will be disabled only when all columns are not in load list // warmup will be disabled only when all columns are not in load list
bool merged_in_load_list = false; bool merged_in_load_list = false;
std::vector<FieldId> milvus_field_ids; std::vector<FieldId> milvus_field_ids;
milvus_field_ids.reserve(field_id_list.size());
for (int i = 0; i < field_id_list.size(); ++i) { for (int i = 0; i < field_id_list.size(); ++i) {
milvus_field_ids.push_back(FieldId(field_id_list.Get(i))); milvus_field_ids.push_back(FieldId(field_id_list.Get(i)));
merged_in_load_list = merged_in_load_list || merged_in_load_list = merged_in_load_list ||

View File

@ -23,9 +23,11 @@ TimestampIndex::build_with(const Timestamp* timestamps, int64_t size) {
auto num_slice = lengths_.size(); auto num_slice = lengths_.size();
Assert(num_slice > 0); Assert(num_slice > 0);
std::vector<int64_t> prefix_sums; std::vector<int64_t> prefix_sums;
prefix_sums.reserve(num_slice + 1);
int offset = 0; int offset = 0;
prefix_sums.push_back(offset); prefix_sums.push_back(offset);
std::vector<Timestamp> timestamp_barriers; std::vector<Timestamp> timestamp_barriers;
timestamp_barriers.reserve(num_slice + 1);
Timestamp last_max_v = 0; Timestamp last_max_v = 0;
for (int slice_id = 0; slice_id < num_slice; ++slice_id) { for (int slice_id = 0; slice_id < num_slice; ++slice_id) {
auto length = lengths_[slice_id]; auto length = lengths_[slice_id];

View File

@ -89,6 +89,7 @@ GroupChunkTranslator::GroupChunkTranslator(
.GetArrowFileSystem(); .GetArrowFileSystem();
// Get row group metadata from files // Get row group metadata from files
row_group_meta_list_.reserve(insert_files_.size());
for (const auto& file : insert_files_) { for (const auto& file : insert_files_) {
auto reader = std::make_shared<milvus_storage::FileRowGroupReader>( auto reader = std::make_shared<milvus_storage::FileRowGroupReader>(
fs, fs,
@ -110,11 +111,16 @@ GroupChunkTranslator::GroupChunkTranslator(
file_row_group_prefix_sum_.reserve(row_group_meta_list_.size() + 1); file_row_group_prefix_sum_.reserve(row_group_meta_list_.size() + 1);
file_row_group_prefix_sum_.push_back( file_row_group_prefix_sum_.push_back(
0); // Base case: 0 row groups before first file 0); // Base case: 0 row groups before first file
size_t total_row_groups = 0;
for (const auto& file_metas : row_group_meta_list_) { for (const auto& file_metas : row_group_meta_list_) {
total_row_groups += file_metas.size();
file_row_group_prefix_sum_.push_back(file_row_group_prefix_sum_.back() + file_row_group_prefix_sum_.push_back(file_row_group_prefix_sum_.back() +
file_metas.size()); file_metas.size());
} }
meta_.num_rows_until_chunk_.reserve(total_row_groups + 1);
meta_.chunk_memory_size_.reserve(total_row_groups);
meta_.num_rows_until_chunk_.push_back(0); meta_.num_rows_until_chunk_.push_back(0);
for (const auto& row_group_meta : row_group_meta_list_) { for (const auto& row_group_meta : row_group_meta_list_) {
for (int i = 0; i < row_group_meta.size(); ++i) { for (int i = 0; i < row_group_meta.size(); ++i) {

View File

@ -355,6 +355,7 @@ struct TantivyIndexWrapper {
int64_t offset_begin) { int64_t offset_begin) {
assert(!finished_); assert(!finished_);
std::vector<const char*> views; std::vector<const char*> views;
views.reserve(len);
for (uintptr_t i = 0; i < len; i++) { for (uintptr_t i = 0; i < len; i++) {
views.push_back(array[i].c_str()); views.push_back(array[i].c_str());
} }
@ -435,6 +436,7 @@ struct TantivyIndexWrapper {
if constexpr (std::is_same_v<T, std::string>) { if constexpr (std::is_same_v<T, std::string>) {
std::vector<const char*> views; std::vector<const char*> views;
views.reserve(len);
for (uintptr_t i = 0; i < len; i++) { for (uintptr_t i = 0; i < len; i++) {
views.push_back(array[i].c_str()); views.push_back(array[i].c_str());
} }
@ -621,6 +623,7 @@ struct TantivyIndexWrapper {
if constexpr (std::is_same_v<T, std::string>) { if constexpr (std::is_same_v<T, std::string>) {
std::vector<const char*> views; std::vector<const char*> views;
views.reserve(len);
for (uintptr_t i = 0; i < len; i++) { for (uintptr_t i = 0; i < len; i++) {
views.push_back(array[i].c_str()); views.push_back(array[i].c_str());
} }
@ -709,6 +712,7 @@ struct TantivyIndexWrapper {
} else { } else {
// smaller integer should be converted first // smaller integer should be converted first
std::vector<int64_t> buf(len); std::vector<int64_t> buf(len);
buf.reserve(len);
for (uintptr_t i = 0; i < len; ++i) { for (uintptr_t i = 0; i < len; ++i) {
buf[i] = static_cast<int64_t>(terms[i]); buf[i] = static_cast<int64_t>(terms[i]);
} }
@ -726,6 +730,7 @@ struct TantivyIndexWrapper {
bitset); bitset);
} else { } else {
std::vector<double> buf(len); std::vector<double> buf(len);
buf.reserve(len);
for (uintptr_t i = 0; i < len; ++i) { for (uintptr_t i = 0; i < len; ++i) {
buf[i] = static_cast<double>(terms[i]); buf[i] = static_cast<double>(terms[i]);
} }
@ -736,6 +741,7 @@ struct TantivyIndexWrapper {
if constexpr (std::is_same_v<T, std::string>) { if constexpr (std::is_same_v<T, std::string>) {
std::vector<const char*> views; std::vector<const char*> views;
views.reserve(len);
for (uintptr_t i = 0; i < len; i++) { for (uintptr_t i = 0; i < len; i++) {
views.push_back(terms[i].c_str()); views.push_back(terms[i].c_str());
} }