admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #492 from fishpenguin/0.6.0-yk-refactor-scheduler

Browse Source 
Add log in scheduler/optimizer and gpu no usage during index build
This commit is contained in:
Jin Hai 2019-11-23 12:03:28 +08:00 committed by GitHub
commit 87eda7b34b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 40 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -20,6 +20,7 @@ Please mark all change in change log and use the ticket from JIRA.
- \#440 - Query API in customization still uses old version
- \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version
- \#458 - Index data is not compatible between 0.5 and 0.6
- \#486 - gpu no usage during index building
## Feature
- \#12 - Pure CPU version for Milvus
@ -28,6 +29,7 @@ Please mark all change in change log and use the ticket from JIRA.
- \#226 - Experimental shards middleware for Milvus
- \#227 - Support new index types SPTAG-KDT and SPTAG-BKT
- \#346 - Support build index with multiple gpu
- \#488 - Add log in scheduler/optimizer
## Improvement
- \#255 - Add ivfsq8 test report detailed version

19
core/src/scheduler/SchedInst.h
View File

@ -106,6 +106,25 @@ class OptimizerInst {
server::Config& config = server::Config::GetInstance();
config.GetGpuResourceConfigEnable(enable_gpu);
if (enable_gpu) {
std::vector<int64_t> build_gpus;
std::vector<int64_t> search_gpus;
int64_t gpu_search_threshold;
config.GetGpuResourceConfigBuildIndexResources(build_gpus);
config.GetGpuResourceConfigSearchResources(search_gpus);
config.GetEngineConfigGpuSearchThreshold(gpu_search_threshold);
std::string build_msg = "Build index gpu:";
for (auto build_id : build_gpus) {
build_msg.append(" gpu" + std::to_string(build_id));
}
SERVER_LOG_DEBUG << build_msg;
std::string search_msg = "Search gpu:";
for (auto search_id : search_gpus) {
search_msg.append(" gpu" + std::to_string(search_id));
}
search_msg.append(". gpu_search_threshold:" + std::to_string(gpu_search_threshold));
SERVER_LOG_DEBUG << search_msg;
pass_list.push_back(std::make_shared<BuildIndexPass>());
pass_list.push_back(std::make_shared<FaissFlatPass>());
pass_list.push_back(std::make_shared<FaissIVFFlatPass>());

8
core/src/scheduler/optimizer/BuildIndexPass.cpp
View File

@ -26,8 +26,7 @@ namespace scheduler {
void
BuildIndexPass::Init() {
server::Config& config = server::Config::GetInstance();
std::vector<int64_t> build_resources;
Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources);
Status s = config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids_);
if (!s.ok()) {
throw;
}
@ -38,13 +37,16 @@ BuildIndexPass::Run(const TaskPtr& task) {
if (task->Type() != TaskType::BuildIndexTask)
return false;
if (build_gpu_ids_.empty())
if (build_gpu_ids_.empty()) {
SERVER_LOG_WARNING << "BuildIndexPass cannot get build index gpu!";
return false;
}
ResourcePtr res_ptr;
res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, build_gpu_ids_[specified_gpu_id_]);
auto label = std::make_shared<SpecResLabel>(std::weak_ptr<Resource>(res_ptr));
task->label() = label;
SERVER_LOG_DEBUG << "Specify gpu" << specified_gpu_id_ << " to build index!";
specified_gpu_id_ = (specified_gpu_id_ + 1) % build_gpu_ids_.size();
return true;

2
core/src/scheduler/optimizer/BuildIndexPass.h
View File

@ -45,7 +45,7 @@ class BuildIndexPass : public Pass {
private:
uint64_t specified_gpu_id_ = 0;
std::vector<int32_t> build_gpu_ids_;
std::vector<int64_t> build_gpu_ids_;
};
using BuildIndexPassPtr = std::shared_ptr<BuildIndexPass>;

2
core/src/scheduler/optimizer/FaissFlatPass.cpp
View File

@ -54,9 +54,11 @@ FaissFlatPass::Run(const TaskPtr& task) {
auto search_job = std::static_pointer_cast<SearchJob>(search_task->job_.lock());
ResourcePtr res_ptr;
if (search_job->nq() < threshold_) {
SERVER_LOG_DEBUG << "FaissFlatPass: nq < gpu_search_threshold, specify cpu to search!";
res_ptr = ResMgrInst::GetInstance()->GetResource("cpu");
} else {
auto best_device_id = count_ % gpus.size();
SERVER_LOG_DEBUG << "FaissFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id << " to search!";
count_++;
res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id);
}

3
core/src/scheduler/optimizer/FaissIVFFlatPass.cpp
View File

@ -54,9 +54,12 @@ FaissIVFFlatPass::Run(const TaskPtr& task) {
auto search_job = std::static_pointer_cast<SearchJob>(search_task->job_.lock());
ResourcePtr res_ptr;
if (search_job->nq() < threshold_) {
SERVER_LOG_DEBUG << "FaissIVFFlatPass: nq < gpu_search_threshold, specify cpu to search!";
res_ptr = ResMgrInst::GetInstance()->GetResource("cpu");
} else {
auto best_device_id = count_ % gpus.size();
SERVER_LOG_DEBUG << "FaissIVFFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id
<< " to search!";
count_++;
res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id);
}

3
core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp
View File

@ -51,9 +51,12 @@ FaissIVFSQ8HPass::Run(const TaskPtr& task) {
auto search_job = std::static_pointer_cast<SearchJob>(search_task->job_.lock());
ResourcePtr res_ptr;
if (search_job->nq() < threshold_) {
SERVER_LOG_DEBUG << "FaissIVFSQ8HPass: nq < gpu_search_threshold, specify cpu to search!";
res_ptr = ResMgrInst::GetInstance()->GetResource("cpu");
} else {
auto best_device_id = count_ % gpus.size();
SERVER_LOG_DEBUG << "FaissIVFSQ8HPass: nq > gpu_search_threshold, specify gpu" << best_device_id
<< " to search!";
count_++;
res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id);
}

3
core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp
View File

@ -54,9 +54,12 @@ FaissIVFSQ8Pass::Run(const TaskPtr& task) {
auto search_job = std::static_pointer_cast<SearchJob>(search_task->job_.lock());
ResourcePtr res_ptr;
if (search_job->nq() < threshold_) {
SERVER_LOG_DEBUG << "FaissIVFSQ8Pass: nq < gpu_search_threshold, specify cpu to search!";
res_ptr = ResMgrInst::GetInstance()->GetResource("cpu");
} else {
auto best_device_id = count_ % gpus.size();
SERVER_LOG_DEBUG << "FaissIVFSQ8Pass: nq > gpu_search_threshold, specify gpu" << best_device_id
<< " to search!";
count_++;
res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id);
}

1
core/src/scheduler/optimizer/FallbackPass.cpp
View File

@ -33,6 +33,7 @@ FallbackPass::Run(const TaskPtr& task) {
return false;
}
// NEVER be empty
SERVER_LOG_DEBUG << "FallbackPass!";
auto cpu = ResMgrInst::GetInstance()->GetCpuResources()[0];
auto label = std::make_shared<SpecResLabel>(cpu);
task->label() = label;

2
core/src/scheduler/task/BuildIndexTask.cpp
View File

@ -85,7 +85,7 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) {
size_t file_size = to_index_engine_->PhysicalSize();
std::string info = "Load file id:" + std::to_string(file_->id_) +
std::string info = "Load file id:" + std::to_string(file_->id_) + " " + type_str +
" file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) +
" bytes from location: " + file_->location_ + " totally cost";
double span = rc.ElapseFromBegin(info);