admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: estimate the size of interim index (#44104)

Browse Source 
issue: #41435

Signed-off-by: cqy123456 <qianya.cheng@zilliz.com>
This commit is contained in:
cqy123456 2025-08-28 19:37:51 +08:00 committed by GitHub
parent 4bc86cb335
commit 844caf5cfe
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 27 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
internal/core/src/segcore/storagev1translator/InterimSealedIndexTranslator.cpp
View File

@ -44,15 +44,34 @@ InterimSealedIndexTranslator::cell_id_of(
milvus::cachinglayer::ResourceUsage
InterimSealedIndexTranslator::estimated_byte_size_of_cell(
milvus::cachinglayer::cid_t cid) const {
// SCANN reference the raw data and has little meta, thus we ignore its size.
if (index_type_ == knowhere::IndexEnum::INDEX_FAISS_SCANN_DVR) {
return milvus::cachinglayer::ResourceUsage{0, 0};
}
// IVF_FLAT_CC, SPARSE_WAND_CC and SPARSE_INVERTED_INDEX_CC basically has the same size as the
// raw data.
// TODO(tiered storage 1) cqy123456: provide a better estimation for IVF_SQ_CC once supported.
auto size = vec_data_->DataByteSize();
return milvus::cachinglayer::ResourceUsage{static_cast<int64_t>(size), 0};
auto row_count = vec_data_->NumRows();
// TODO: hack, move these estimate logic to knowhere
// ignore the size of centroids
if (index_type_ == knowhere::IndexEnum::INDEX_FAISS_SCANN_DVR) {
auto vec_size = size_t(index::GetValueFromConfig<int>(
build_config_, knowhere::indexparam::SUB_DIM)
.value() /
8 * dim_);
if (build_config_[knowhere::indexparam::REFINE_TYPE] ==
knowhere::RefineType::UINT8_QUANT) {
vec_size += dim_ * 1;
} else if (build_config_[knowhere::indexparam::REFINE_TYPE] ==
knowhere::RefineType::FLOAT16_QUANT ||
build_config_[knowhere::indexparam::REFINE_TYPE] ==
knowhere::RefineType::BFLOAT16_QUANT) {
vec_size += dim_ * 2;
} // else knowhere::RefineType::DATA_VIEW, no extra size
return milvus::cachinglayer::ResourceUsage{vec_size * row_count, 0};
} else if (index_type_ == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC) {
// fp16/bf16 all use float32 to build index
return milvus::cachinglayer::ResourceUsage{
row_count * sizeof(float) * dim_, 0};
} else {
// SPARSE_WAND_CC and SPARSE_INVERTED_INDEX_CC basically has the same size as the
// raw data.
return milvus::cachinglayer::ResourceUsage{size, 0};
}
}
const std::string&