admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: [2.6] add vector reserve to improve memory allocation in segcore (#45759)

Browse Source 
This commit optimizes std::vector usage across segcore by adding
reserve() calls where the size is known in advance, reducing memory
reallocations during push_back operations.

Changes:

TimestampIndex.cpp: Reserve space for prefix_sums and timestamp_barriers
SegmentGrowingImpl.cpp: Reserve space for binlog info vectors
ChunkedSegmentSealedImpl.cpp: Reserve space for futures and field data
vectors
storagev2translator/GroupChunkTranslator.cpp: Reserve space for metadata
vectors
This improves performance by avoiding multiple memory reallocations when
the vector size is predictable.

issue: https://github.com/milvus-io/milvus/issues/45679
pr: https://github.com/milvus-io/milvus/pull/45757

Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>
This commit is contained in:
Buqian Zheng 2025-11-26 10:33:09 +08:00 committed by GitHub
parent feffbf90d2
commit ce302c1555
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 15 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
internal/core/src/segcore/ChunkedSegmentSealedImpl.cpp
View File

@ -285,6 +285,7 @@ ChunkedSegmentSealedImpl::load_column_group_data_internal(
// warmup will be disabled only when all columns are not in load list
bool merged_in_load_list = false;
std::vector<FieldId> milvus_field_ids;
milvus_field_ids.reserve(field_id_list.size());
for (int i = 0; i < field_id_list.size(); ++i) {
milvus_field_ids.push_back(FieldId(field_id_list.Get(i)));
merged_in_load_list = merged_in_load_list ||

2
internal/core/src/segcore/TimestampIndex.cpp
View File

@ -23,9 +23,11 @@ TimestampIndex::build_with(const Timestamp* timestamps, int64_t size) {
auto num_slice = lengths_.size();
Assert(num_slice > 0);
std::vector<int64_t> prefix_sums;
prefix_sums.reserve(num_slice + 1);
int offset = 0;
prefix_sums.push_back(offset);
std::vector<Timestamp> timestamp_barriers;
timestamp_barriers.reserve(num_slice + 1);
Timestamp last_max_v = 0;
for (int slice_id = 0; slice_id < num_slice; ++slice_id) {
auto length = lengths_[slice_id];

6
internal/core/src/segcore/storagev2translator/GroupChunkTranslator.cpp
View File

@ -89,6 +89,7 @@ GroupChunkTranslator::GroupChunkTranslator(
.GetArrowFileSystem();
// Get row group metadata from files
row_group_meta_list_.reserve(insert_files_.size());
for (const auto& file : insert_files_) {
auto reader = std::make_shared<milvus_storage::FileRowGroupReader>(
fs,
@ -110,11 +111,16 @@ GroupChunkTranslator::GroupChunkTranslator(
file_row_group_prefix_sum_.reserve(row_group_meta_list_.size() + 1);
file_row_group_prefix_sum_.push_back(
0); // Base case: 0 row groups before first file
size_t total_row_groups = 0;
for (const auto& file_metas : row_group_meta_list_) {
total_row_groups += file_metas.size();
file_row_group_prefix_sum_.push_back(file_row_group_prefix_sum_.back() +
file_metas.size());
}
meta_.num_rows_until_chunk_.reserve(total_row_groups + 1);
meta_.chunk_memory_size_.reserve(total_row_groups);
meta_.num_rows_until_chunk_.push_back(0);
for (const auto& row_group_meta : row_group_meta_list_) {
for (int i = 0; i < row_group_meta.size(); ++i) {

6
internal/core/thirdparty/tantivy/tantivy-wrapper.h vendored
View File

@ -355,6 +355,7 @@ struct TantivyIndexWrapper {
int64_t offset_begin) {
assert(!finished_);
std::vector<const char*> views;
views.reserve(len);
for (uintptr_t i = 0; i < len; i++) {
views.push_back(array[i].c_str());
}
@ -435,6 +436,7 @@ struct TantivyIndexWrapper {
if constexpr (std::is_same_v<T, std::string>) {
std::vector<const char*> views;
views.reserve(len);
for (uintptr_t i = 0; i < len; i++) {
views.push_back(array[i].c_str());
}
@ -621,6 +623,7 @@ struct TantivyIndexWrapper {
if constexpr (std::is_same_v<T, std::string>) {
std::vector<const char*> views;
views.reserve(len);
for (uintptr_t i = 0; i < len; i++) {
views.push_back(array[i].c_str());
}
@ -709,6 +712,7 @@ struct TantivyIndexWrapper {
} else {
// smaller integer should be converted first
std::vector<int64_t> buf(len);
buf.reserve(len);
for (uintptr_t i = 0; i < len; ++i) {
buf[i] = static_cast<int64_t>(terms[i]);
}
@ -726,6 +730,7 @@ struct TantivyIndexWrapper {
bitset);
} else {
std::vector<double> buf(len);
buf.reserve(len);
for (uintptr_t i = 0; i < len; ++i) {
buf[i] = static_cast<double>(terms[i]);
}
@ -736,6 +741,7 @@ struct TantivyIndexWrapper {
if constexpr (std::is_same_v<T, std::string>) {
std::vector<const char*> views;
views.reserve(len);
for (uintptr_t i = 0; i < len; i++) {
views.push_back(terms[i].c_str());
}