From b68f4f43c1ecf9671876658907c68673bbe46ade Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Sat, 23 Nov 2019 10:29:11 +0800 Subject: [PATCH 1/3] Add log in scheduler/optimizer --- CHANGELOG.md | 1 + core/src/scheduler/optimizer/BuildIndexPass.cpp | 4 +++- core/src/scheduler/optimizer/FaissFlatPass.cpp | 2 ++ core/src/scheduler/optimizer/FaissIVFFlatPass.cpp | 3 +++ core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp | 3 +++ core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp | 3 +++ core/src/scheduler/optimizer/FallbackPass.cpp | 1 + 7 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f08ca3bf42..7a518a8d41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#226 - Experimental shards middleware for Milvus - \#227 - Support new index types SPTAG-KDT and SPTAG-BKT - \#346 - Support build index with multiple gpu +- \#488 - Add log in scheduler/optimizer ## Improvement - \#255 - Add ivfsq8 test report detailed version diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp index d535b9675f..faa451bc56 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -38,8 +38,10 @@ BuildIndexPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::BuildIndexTask) return false; - if (build_gpu_ids_.empty()) + if (build_gpu_ids_.empty()) { + SERVER_LOG_WARNING << "BUildIndexPass cannot get build index gpu!"; return false; + } ResourcePtr res_ptr; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, build_gpu_ids_[specified_gpu_id_]); diff --git a/core/src/scheduler/optimizer/FaissFlatPass.cpp b/core/src/scheduler/optimizer/FaissFlatPass.cpp index 61ca1b9ec9..c78f7d57e1 100644 --- a/core/src/scheduler/optimizer/FaissFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissFlatPass.cpp @@ -54,9 +54,11 @@ FaissFlatPass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (search_job->nq() < threshold_) { + SERVER_LOG_DEBUG << "FaissFlatPass: nq < gpu_search_threshold, specify cpu to search!"; res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % gpus.size(); + SERVER_LOG_DEBUG << "FaissFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id << " to search!"; count_++; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp index 1f1efb374b..17067cf24e 100644 --- a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp @@ -54,9 +54,12 @@ FaissIVFFlatPass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (search_job->nq() < threshold_) { + SERVER_LOG_DEBUG << "FaissIVFFlatPass: nq < gpu_search_threshold, specify cpu to search!"; res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % gpus.size(); + SERVER_LOG_DEBUG << "FaissIVFFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id + << " to search!"; count_++; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index a99e861e03..676ed1720e 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -51,9 +51,12 @@ FaissIVFSQ8HPass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (search_job->nq() < threshold_) { + SERVER_LOG_DEBUG << "FaissIVFSQ8HPass: nq < gpu_search_threshold, specify cpu to search!"; res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % gpus.size(); + SERVER_LOG_DEBUG << "FaissIVFSQ8HPass: nq > gpu_search_threshold, specify gpu" << best_device_id + << " to search!"; count_++; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp index 30dd306b3b..4f06c1e1fc 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp @@ -54,9 +54,12 @@ FaissIVFSQ8Pass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (search_job->nq() < threshold_) { + SERVER_LOG_DEBUG << "FaissIVFSQ8Pass: nq < gpu_search_threshold, specify cpu to search!"; res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % gpus.size(); + SERVER_LOG_DEBUG << "FaissIVFSQ8Pass: nq > gpu_search_threshold, specify gpu" << best_device_id + << " to search!"; count_++; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } diff --git a/core/src/scheduler/optimizer/FallbackPass.cpp b/core/src/scheduler/optimizer/FallbackPass.cpp index 2e275ede4b..15687bb9bb 100644 --- a/core/src/scheduler/optimizer/FallbackPass.cpp +++ b/core/src/scheduler/optimizer/FallbackPass.cpp @@ -33,6 +33,7 @@ FallbackPass::Run(const TaskPtr& task) { return false; } // NEVER be empty + SERVER_LOG_DEBUG << "FallbackPass!"; auto cpu = ResMgrInst::GetInstance()->GetCpuResources()[0]; auto label = std::make_shared(cpu); task->label() = label; From 288490f980f279e5ebe74735e06e3845054aca85 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Sat, 23 Nov 2019 11:07:42 +0800 Subject: [PATCH 2/3] gpu no usage during index building --- CHANGELOG.md | 1 + core/src/scheduler/optimizer/BuildIndexPass.cpp | 4 ++-- core/src/scheduler/optimizer/BuildIndexPass.h | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a518a8d41..ccbbe9f64e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#440 - Query API in customization still uses old version - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version - \#458 - Index data is not compatible between 0.5 and 0.6 +- \#486 - gpu no usage during index building ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp index faa451bc56..55e28ec672 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -26,8 +26,7 @@ namespace scheduler { void BuildIndexPass::Init() { server::Config& config = server::Config::GetInstance(); - std::vector build_resources; - Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources); + Status s = config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids_); if (!s.ok()) { throw; } @@ -47,6 +46,7 @@ BuildIndexPass::Run(const TaskPtr& task) { res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, build_gpu_ids_[specified_gpu_id_]); auto label = std::make_shared(std::weak_ptr(res_ptr)); task->label() = label; + SERVER_LOG_DEBUG << "Specify gpu" << specified_gpu_id_ << " to build index!"; specified_gpu_id_ = (specified_gpu_id_ + 1) % build_gpu_ids_.size(); return true; diff --git a/core/src/scheduler/optimizer/BuildIndexPass.h b/core/src/scheduler/optimizer/BuildIndexPass.h index 4f7117fc4e..3adf1259a7 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.h +++ b/core/src/scheduler/optimizer/BuildIndexPass.h @@ -45,7 +45,7 @@ class BuildIndexPass : public Pass { private: uint64_t specified_gpu_id_ = 0; - std::vector build_gpu_ids_; + std::vector build_gpu_ids_; }; using BuildIndexPassPtr = std::shared_ptr; From 8a4cc9263e06708299e74261be8c23f0d9aa3412 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Sat, 23 Nov 2019 11:56:34 +0800 Subject: [PATCH 3/3] [skip ci]Add more log msg --- core/src/scheduler/SchedInst.h | 19 +++++++++++++++++++ .../scheduler/optimizer/BuildIndexPass.cpp | 2 +- core/src/scheduler/task/BuildIndexTask.cpp | 2 +- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index 6273af7a9f..1e8a7acf2e 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -106,6 +106,25 @@ class OptimizerInst { server::Config& config = server::Config::GetInstance(); config.GetGpuResourceConfigEnable(enable_gpu); if (enable_gpu) { + std::vector build_gpus; + std::vector search_gpus; + int64_t gpu_search_threshold; + config.GetGpuResourceConfigBuildIndexResources(build_gpus); + config.GetGpuResourceConfigSearchResources(search_gpus); + config.GetEngineConfigGpuSearchThreshold(gpu_search_threshold); + std::string build_msg = "Build index gpu:"; + for (auto build_id : build_gpus) { + build_msg.append(" gpu" + std::to_string(build_id)); + } + SERVER_LOG_DEBUG << build_msg; + + std::string search_msg = "Search gpu:"; + for (auto search_id : search_gpus) { + search_msg.append(" gpu" + std::to_string(search_id)); + } + search_msg.append(". gpu_search_threshold:" + std::to_string(gpu_search_threshold)); + SERVER_LOG_DEBUG << search_msg; + pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp index 55e28ec672..5e5719a1bd 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -38,7 +38,7 @@ BuildIndexPass::Run(const TaskPtr& task) { return false; if (build_gpu_ids_.empty()) { - SERVER_LOG_WARNING << "BUildIndexPass cannot get build index gpu!"; + SERVER_LOG_WARNING << "BuildIndexPass cannot get build index gpu!"; return false; } diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index f561fa947d..e952bd0938 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -85,7 +85,7 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) { size_t file_size = to_index_engine_->PhysicalSize(); - std::string info = "Load file id:" + std::to_string(file_->id_) + + std::string info = "Load file id:" + std::to_string(file_->id_) + " " + type_str + " file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + " bytes from location: " + file_->location_ + " totally cost"; double span = rc.ElapseFromBegin(info);