From 7512cc1e062aa3d920fda2be33949f0dd4851eb6 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Wed, 3 Jul 2019 20:07:11 +0800 Subject: [PATCH 1/3] acc test Former-commit-id: a7803568fa785d403e7c92b83c66f9f47f7c8f13 --- cpp/src/db/DBMetaImpl.cpp | 14 ++++++++------ cpp/src/db/FaissExecutionEngine.cpp | 1 + 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 8c56c863e7..d13899dca0 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -612,7 +612,8 @@ Status DBMetaImpl::GetTableFiles(const std::string& table_id, TableFilesSchema& table_files) { try { table_files.clear(); - auto files = ConnectorPtr->select(columns(&TableFileSchema::file_id_, + auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, + &TableFileSchema::file_id_, &TableFileSchema::file_type_, &TableFileSchema::size_, &TableFileSchema::date_, @@ -631,11 +632,12 @@ Status DBMetaImpl::GetTableFiles(const std::string& table_id, for (auto &file : files) { TableFileSchema file_schema; file_schema.table_id_ = table_id; - file_schema.file_id_ = std::get<0>(file); - file_schema.file_type_ = std::get<1>(file); - file_schema.size_ = std::get<2>(file); - file_schema.date_ = std::get<3>(file); - file_schema.engine_type_ = std::get<4>(file); + file_schema.id_ = std::get<0>(file); + file_schema.file_id_ = std::get<1>(file); + file_schema.file_type_ = std::get<2>(file); + file_schema.size_ = std::get<3>(file); + file_schema.date_ = std::get<4>(file); + file_schema.engine_type_ = std::get<5>(file); file_schema.dimension_ = table_schema.dimension_; GetTableFilePath(file_schema); diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index 20bd530e78..201c07dbcf 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -138,6 +138,7 @@ Status FaissExecutionEngine::Search(long n, auto start_time = METRICS_NOW_TIME; std::shared_ptr ivf_index = std::dynamic_pointer_cast(pIndex_); + //ENGINE_LOG_DEBUG << "Index nlist: " << ivf_index->nlist << ", ntotal: "<< ivf_index->ntotal; if(ivf_index) { ENGINE_LOG_DEBUG << "Index type: IVFFLAT nProbe: " << nprobe_; ivf_index->nprobe = nprobe_; From 55afa772d3dc34ebbbdd093dacc4bd3ad900ad58 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Wed, 3 Jul 2019 20:59:36 +0800 Subject: [PATCH 2/3] ADD LOG Former-commit-id: 282768be582212cd6bf364ea1903a9ad716106c0 --- cpp/src/db/scheduler/task/SearchTask.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index d04f270331..80c7291753 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -167,6 +167,14 @@ std::shared_ptr SearchTask::Execute() { ClusterResult(output_ids, output_distence, context->nq(), inner_k, result_set); rc.Record("cluster result"); + + SERVER_LOG_DEBUG << "Query Result: "; + for(auto& id2score_vector: result_set) { + for(auto& pair: id2score_vector) { + SERVER_LOG_DEBUG << "id: " << pair.first << ", distance: " << pair.second; + } + } + //step 4: pick up topk result TopkResult(result_set, inner_k, context->GetResult()); rc.Record("reduce topk"); From 45c172543f3ccdc05bfd55ec0fe6e271819e8b7c Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Thu, 4 Jul 2019 12:41:24 +0800 Subject: [PATCH 3/3] MS-151 Fix topk problem Former-commit-id: eeac049095172303c32f4f26ee13b9c8724870b0 --- cpp/src/db/FaissExecutionEngine.cpp | 1 - cpp/src/db/scheduler/task/SearchTask.cpp | 13 +++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index 201c07dbcf..20bd530e78 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -138,7 +138,6 @@ Status FaissExecutionEngine::Search(long n, auto start_time = METRICS_NOW_TIME; std::shared_ptr ivf_index = std::dynamic_pointer_cast(pIndex_); - //ENGINE_LOG_DEBUG << "Index nlist: " << ivf_index->nlist << ", ntotal: "<< ivf_index->ntotal; if(ivf_index) { ENGINE_LOG_DEBUG << "Index type: IVFFLAT nProbe: " << nprobe_; ivf_index->nprobe = nprobe_; diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index 80c7291753..2bfac90e20 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -151,7 +151,7 @@ std::shared_ptr SearchTask::Execute() { std::vector output_distence; for(auto& context : search_contexts_) { //step 1: allocate memory - auto inner_k = index_engine_->Count() < context->topk() ? index_engine_->Count() : context->topk(); + auto inner_k = context->topk(); output_ids.resize(inner_k*context->nq()); output_distence.resize(inner_k*context->nq()); @@ -164,17 +164,10 @@ std::shared_ptr SearchTask::Execute() { //step 3: cluster result SearchContext::ResultSet result_set; - ClusterResult(output_ids, output_distence, context->nq(), inner_k, result_set); + auto spec_k = index_engine_->Count() < context->topk() ? index_engine_->Count() : context->topk(); + ClusterResult(output_ids, output_distence, context->nq(), spec_k, result_set); rc.Record("cluster result"); - - SERVER_LOG_DEBUG << "Query Result: "; - for(auto& id2score_vector: result_set) { - for(auto& pair: id2score_vector) { - SERVER_LOG_DEBUG << "id: " << pair.first << ", distance: " << pair.second; - } - } - //step 4: pick up topk result TopkResult(result_set, inner_k, context->GetResult()); rc.Record("reduce topk");