diff --git a/CHANGELOG.md b/CHANGELOG.md index 957821ff4c..73568d4292 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#458 - Index data is not compatible between 0.5 and 0.6 - \#465 - Server hang caused by searching with nsg index - \#486 - gpu no usage during index building +- \#509 - IVF_PQ index build trapped into dead loop caused by invalid params - \#513 - Unittest DELETE_BY_RANGE sometimes failed ## Feature diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index aa4b3c12b8..d49747d8f4 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -22,6 +22,7 @@ #include #include +#include // TODO(lxj): add conf checker @@ -129,17 +130,35 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { conf->metric_type = metaconf.metric_type; conf->gpu_id = metaconf.gpu_id; conf->nbits = 8; - - if (!(conf->d % 4)) - conf->m = conf->d / 4; // compression radio = 16 - else if (!(conf->d % 2)) - conf->m = conf->d / 2; // compression radio = 8 - else if (!(conf->d % 3)) - conf->m = conf->d / 3; // compression radio = 12 - else - conf->m = conf->d; // same as SQ8, compression radio = 4 - MatchBase(conf); + + /* + * Faiss 1.6 + * Only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims per sub-quantizer are currently supporte with + * no precomputed codes. Precomputed codes supports any number of dimensions, but will involve memory overheads. + */ + static std::vector support_dim_per_subquantizer{32, 28, 24, 20, 16, 12, 10, 8, 6, 4, 3, 2, 1}; + static std::vector support_subquantizer{96, 64, 56, 48, 40, 32, 28, 24, 20, 16, 12, 8, 4, 3, 2, 1}; + std::vector resset; + for (const auto& dimperquantizer : support_dim_per_subquantizer) { + if (!(conf->d % dimperquantizer)) { + auto subquantzier_num = conf->d / dimperquantizer; + auto finder = std::find(support_subquantizer.begin(), support_subquantizer.end(), subquantzier_num); + if (finder != support_subquantizer.end()) { + resset.push_back(subquantzier_num); + } + } + } + + if (resset.empty()) { + // todo(linxj): throw exception here. + return nullptr; + } + static int64_t compression_level = 1; // 1:low, 2:high + if (compression_level == 1) { + conf->m = resset[int(resset.size() / 2)]; + WRAPPER_LOG_DEBUG << "PQ m = " << conf->m << ", compression radio = " << conf->d / conf->m * 4; + } return conf; }