admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(db): cache for search

Browse Source 
Former-commit-id: 8e13efdaa593b4ebb83b06d33d5127180d756597
This commit is contained in:
Xu Peng 2019-04-24 16:20:36 +08:00
parent 4e59f966c3
commit a6369c7a4e
1 changed files with 13 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cpp/src/db/DBImpl.cpp
View File

@ -86,7 +86,13 @@ Status DBImpl::search(const std::string &group_id, size_t k, size_t nq,
// merge raw files and build flat index.
faiss::Index *index(faiss::index_factory(dim, "IDMap,Flat"));
for (auto &file : raw_files) {
auto file_index = dynamic_cast<faiss::IndexIDMap *>(faiss::read_index(file.location.c_str()));
auto to_merge = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(file.location);
if (!to_merge) {
LOG(DEBUG) << "Disk io from: " << file.location;
to_merge = read_index(file.location.c_str());
zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->InsertItem(file.location, to_merge);
}
auto file_index = dynamic_cast<faiss::IndexIDMap *>(to_merge->data().get());
index->add_with_ids(file_index->ntotal,
dynamic_cast<faiss::IndexFlat *>(file_index->index)->xb.data(),
file_index->id_map.data());
@ -123,7 +129,12 @@ Status DBImpl::search(const std::string &group_id, size_t k, size_t nq,
// Search in index file
for (auto &file : index_files) {
auto index = read_index(file.location.c_str());
auto index = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(file.location);
if (!index) {
LOG(DEBUG) << "Disk io from: " << file.location;
index = read_index(file.location.c_str());
zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->InsertItem(file.location, index);
}
index->search(nq, vectors, k, output_distence, output_ids);
cluster(output_ids, output_distence); // cluster to each query
memset(output_distence, 0, k * nq * sizeof(float));