diff --git a/CHANGELOG.md b/CHANGELOG.md index f08ca3bf42..ccbbe9f64e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#440 - Query API in customization still uses old version - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version - \#458 - Index data is not compatible between 0.5 and 0.6 +- \#486 - gpu no usage during index building ## Feature - \#12 - Pure CPU version for Milvus @@ -28,6 +29,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#226 - Experimental shards middleware for Milvus - \#227 - Support new index types SPTAG-KDT and SPTAG-BKT - \#346 - Support build index with multiple gpu +- \#488 - Add log in scheduler/optimizer ## Improvement - \#255 - Add ivfsq8 test report detailed version diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index 6273af7a9f..1e8a7acf2e 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -106,6 +106,25 @@ class OptimizerInst { server::Config& config = server::Config::GetInstance(); config.GetGpuResourceConfigEnable(enable_gpu); if (enable_gpu) { + std::vector build_gpus; + std::vector search_gpus; + int64_t gpu_search_threshold; + config.GetGpuResourceConfigBuildIndexResources(build_gpus); + config.GetGpuResourceConfigSearchResources(search_gpus); + config.GetEngineConfigGpuSearchThreshold(gpu_search_threshold); + std::string build_msg = "Build index gpu:"; + for (auto build_id : build_gpus) { + build_msg.append(" gpu" + std::to_string(build_id)); + } + SERVER_LOG_DEBUG << build_msg; + + std::string search_msg = "Search gpu:"; + for (auto search_id : search_gpus) { + search_msg.append(" gpu" + std::to_string(search_id)); + } + search_msg.append(". gpu_search_threshold:" + std::to_string(gpu_search_threshold)); + SERVER_LOG_DEBUG << search_msg; + pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp index d535b9675f..5e5719a1bd 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -26,8 +26,7 @@ namespace scheduler { void BuildIndexPass::Init() { server::Config& config = server::Config::GetInstance(); - std::vector build_resources; - Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources); + Status s = config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids_); if (!s.ok()) { throw; } @@ -38,13 +37,16 @@ BuildIndexPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::BuildIndexTask) return false; - if (build_gpu_ids_.empty()) + if (build_gpu_ids_.empty()) { + SERVER_LOG_WARNING << "BuildIndexPass cannot get build index gpu!"; return false; + } ResourcePtr res_ptr; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, build_gpu_ids_[specified_gpu_id_]); auto label = std::make_shared(std::weak_ptr(res_ptr)); task->label() = label; + SERVER_LOG_DEBUG << "Specify gpu" << specified_gpu_id_ << " to build index!"; specified_gpu_id_ = (specified_gpu_id_ + 1) % build_gpu_ids_.size(); return true; diff --git a/core/src/scheduler/optimizer/BuildIndexPass.h b/core/src/scheduler/optimizer/BuildIndexPass.h index 4f7117fc4e..3adf1259a7 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.h +++ b/core/src/scheduler/optimizer/BuildIndexPass.h @@ -45,7 +45,7 @@ class BuildIndexPass : public Pass { private: uint64_t specified_gpu_id_ = 0; - std::vector build_gpu_ids_; + std::vector build_gpu_ids_; }; using BuildIndexPassPtr = std::shared_ptr; diff --git a/core/src/scheduler/optimizer/FaissFlatPass.cpp b/core/src/scheduler/optimizer/FaissFlatPass.cpp index 61ca1b9ec9..c78f7d57e1 100644 --- a/core/src/scheduler/optimizer/FaissFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissFlatPass.cpp @@ -54,9 +54,11 @@ FaissFlatPass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (search_job->nq() < threshold_) { + SERVER_LOG_DEBUG << "FaissFlatPass: nq < gpu_search_threshold, specify cpu to search!"; res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % gpus.size(); + SERVER_LOG_DEBUG << "FaissFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id << " to search!"; count_++; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp index 1f1efb374b..17067cf24e 100644 --- a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp @@ -54,9 +54,12 @@ FaissIVFFlatPass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (search_job->nq() < threshold_) { + SERVER_LOG_DEBUG << "FaissIVFFlatPass: nq < gpu_search_threshold, specify cpu to search!"; res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % gpus.size(); + SERVER_LOG_DEBUG << "FaissIVFFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id + << " to search!"; count_++; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index a99e861e03..676ed1720e 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -51,9 +51,12 @@ FaissIVFSQ8HPass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (search_job->nq() < threshold_) { + SERVER_LOG_DEBUG << "FaissIVFSQ8HPass: nq < gpu_search_threshold, specify cpu to search!"; res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % gpus.size(); + SERVER_LOG_DEBUG << "FaissIVFSQ8HPass: nq > gpu_search_threshold, specify gpu" << best_device_id + << " to search!"; count_++; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp index 30dd306b3b..4f06c1e1fc 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp @@ -54,9 +54,12 @@ FaissIVFSQ8Pass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (search_job->nq() < threshold_) { + SERVER_LOG_DEBUG << "FaissIVFSQ8Pass: nq < gpu_search_threshold, specify cpu to search!"; res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % gpus.size(); + SERVER_LOG_DEBUG << "FaissIVFSQ8Pass: nq > gpu_search_threshold, specify gpu" << best_device_id + << " to search!"; count_++; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } diff --git a/core/src/scheduler/optimizer/FallbackPass.cpp b/core/src/scheduler/optimizer/FallbackPass.cpp index 2e275ede4b..15687bb9bb 100644 --- a/core/src/scheduler/optimizer/FallbackPass.cpp +++ b/core/src/scheduler/optimizer/FallbackPass.cpp @@ -33,6 +33,7 @@ FallbackPass::Run(const TaskPtr& task) { return false; } // NEVER be empty + SERVER_LOG_DEBUG << "FallbackPass!"; auto cpu = ResMgrInst::GetInstance()->GetCpuResources()[0]; auto label = std::make_shared(cpu); task->label() = label; diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index f561fa947d..e952bd0938 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -85,7 +85,7 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) { size_t file_size = to_index_engine_->PhysicalSize(); - std::string info = "Load file id:" + std::to_string(file_->id_) + + std::string info = "Load file id:" + std::to_string(file_->id_) + " " + type_str + " file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + " bytes from location: " + file_->location_ + " totally cost"; double span = rc.ElapseFromBegin(info);