From 844caf5cfecddc3448cc2044e8eba91bbfe0f88b Mon Sep 17 00:00:00 2001 From: cqy123456 <39671710+cqy123456@users.noreply.github.com> Date: Thu, 28 Aug 2025 19:37:51 +0800 Subject: [PATCH] enhance: estimate the size of interim index (#44104) issue: #41435 Signed-off-by: cqy123456 --- .../InterimSealedIndexTranslator.cpp | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/internal/core/src/segcore/storagev1translator/InterimSealedIndexTranslator.cpp b/internal/core/src/segcore/storagev1translator/InterimSealedIndexTranslator.cpp index 952b128cd7..51f7c6b697 100644 --- a/internal/core/src/segcore/storagev1translator/InterimSealedIndexTranslator.cpp +++ b/internal/core/src/segcore/storagev1translator/InterimSealedIndexTranslator.cpp @@ -44,15 +44,34 @@ InterimSealedIndexTranslator::cell_id_of( milvus::cachinglayer::ResourceUsage InterimSealedIndexTranslator::estimated_byte_size_of_cell( milvus::cachinglayer::cid_t cid) const { - // SCANN reference the raw data and has little meta, thus we ignore its size. - if (index_type_ == knowhere::IndexEnum::INDEX_FAISS_SCANN_DVR) { - return milvus::cachinglayer::ResourceUsage{0, 0}; - } - // IVF_FLAT_CC, SPARSE_WAND_CC and SPARSE_INVERTED_INDEX_CC basically has the same size as the - // raw data. - // TODO(tiered storage 1) cqy123456: provide a better estimation for IVF_SQ_CC once supported. auto size = vec_data_->DataByteSize(); - return milvus::cachinglayer::ResourceUsage{static_cast(size), 0}; + auto row_count = vec_data_->NumRows(); + // TODO: hack, move these estimate logic to knowhere + // ignore the size of centroids + if (index_type_ == knowhere::IndexEnum::INDEX_FAISS_SCANN_DVR) { + auto vec_size = size_t(index::GetValueFromConfig( + build_config_, knowhere::indexparam::SUB_DIM) + .value() / + 8 * dim_); + if (build_config_[knowhere::indexparam::REFINE_TYPE] == + knowhere::RefineType::UINT8_QUANT) { + vec_size += dim_ * 1; + } else if (build_config_[knowhere::indexparam::REFINE_TYPE] == + knowhere::RefineType::FLOAT16_QUANT || + build_config_[knowhere::indexparam::REFINE_TYPE] == + knowhere::RefineType::BFLOAT16_QUANT) { + vec_size += dim_ * 2; + } // else knowhere::RefineType::DATA_VIEW, no extra size + return milvus::cachinglayer::ResourceUsage{vec_size * row_count, 0}; + } else if (index_type_ == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC) { + // fp16/bf16 all use float32 to build index + return milvus::cachinglayer::ResourceUsage{ + row_count * sizeof(float) * dim_, 0}; + } else { + // SPARSE_WAND_CC and SPARSE_INVERTED_INDEX_CC basically has the same size as the + // raw data. + return milvus::cachinglayer::ResourceUsage{size, 0}; + } } const std::string&