From 994274a1301e307e54e521562ffa780d85f33402 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 14 Oct 2019 11:44:56 +0800 Subject: [PATCH] merge 0.5.0 branch Former-commit-id: c687efe7864690004aecb296c1b75ce7084eb315 --- core/src/CMakeLists.txt | 4 +- core/src/db/engine/ExecutionEngineImpl.cpp | 8 +- core/src/db/engine/ExecutionEngineImpl.h | 2 +- core/src/db/meta/MetaTypes.h | 5 ++ core/src/scheduler/BuildMgr.cpp | 22 ++++++ core/src/scheduler/BuildMgr.h | 63 ++++++++++++++++ .../src/scheduler/optimizer/LargeSQ8HPass.cpp | 73 +++++++++++++++++++ core/src/scheduler/optimizer/LargeSQ8HPass.h | 47 ++++++++++++ 8 files changed, 217 insertions(+), 7 deletions(-) create mode 100644 core/src/scheduler/BuildMgr.cpp create mode 100644 core/src/scheduler/BuildMgr.h create mode 100644 core/src/scheduler/optimizer/LargeSQ8HPass.cpp create mode 100644 core/src/scheduler/optimizer/LargeSQ8HPass.h diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 44b8fc1a5c..b119a517d1 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -182,8 +182,8 @@ target_link_libraries(milvus_server install(TARGETS milvus_server DESTINATION bin) install(FILES - ${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX} - ${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2 + ${CMAKE_SOURCE_DIR}/src/index/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX} + ${CMAKE_SOURCE_DIR}/src/index/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 3fa68aae52..1d758f38fa 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -25,14 +25,14 @@ #include "knowhere/common/Config.h" #include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexIVFSQHybrid.h" +#include "scheduler/Utils.h" #include "server/Config.h" -#include "src/wrapper/VecImpl.h" -#include "src/wrapper/VecIndex.h" #include "wrapper/ConfAdapter.h" #include "wrapper/ConfAdapterMgr.h" +#include "wrapper/VecImpl.h" +#include "wrapper/VecIndex.h" -#include -#include #include #include #include diff --git a/core/src/db/engine/ExecutionEngineImpl.h b/core/src/db/engine/ExecutionEngineImpl.h index 9cbabb2bd5..10379d1651 100644 --- a/core/src/db/engine/ExecutionEngineImpl.h +++ b/core/src/db/engine/ExecutionEngineImpl.h @@ -18,7 +18,7 @@ #pragma once #include "ExecutionEngine.h" -#include "src/wrapper/VecIndex.h" +#include "wrapper/VecIndex.h" #include #include diff --git a/core/src/db/meta/MetaTypes.h b/core/src/db/meta/MetaTypes.h index c973f3fdea..c6a6b6ae87 100644 --- a/core/src/db/meta/MetaTypes.h +++ b/core/src/db/meta/MetaTypes.h @@ -41,6 +41,11 @@ using DateT = int; const DateT EmptyDate = -1; using DatesT = std::vector; +struct DateRange { + DateT start_date_ = 0x1 << 32; + DateT end_date_ = 0; +}; + struct TableSchema { typedef enum { NORMAL, diff --git a/core/src/scheduler/BuildMgr.cpp b/core/src/scheduler/BuildMgr.cpp new file mode 100644 index 0000000000..d90a074d30 --- /dev/null +++ b/core/src/scheduler/BuildMgr.cpp @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/BuildMgr.h" + +namespace milvus { +namespace scheduler {} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/BuildMgr.h b/core/src/scheduler/BuildMgr.h new file mode 100644 index 0000000000..ee7ab38e25 --- /dev/null +++ b/core/src/scheduler/BuildMgr.h @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace milvus { +namespace scheduler { + +class BuildMgr { + public: + explicit BuildMgr(int64_t numoftasks) : numoftasks_(numoftasks) { + } + + public: + void + Put() { + ++numoftasks_; + } + + void + take() { + --numoftasks_; + } + + int64_t + numoftasks() { + return (int64_t)numoftasks_; + } + + private: + std::atomic_long numoftasks_; +}; + +using BuildMgrPtr = std::shared_ptr; + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp b/core/src/scheduler/optimizer/LargeSQ8HPass.cpp new file mode 100644 index 0000000000..62d0e57902 --- /dev/null +++ b/core/src/scheduler/optimizer/LargeSQ8HPass.cpp @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/LargeSQ8HPass.h" +#include "cache/GpuCacheMgr.h" +#include "scheduler/SchedInst.h" +#include "scheduler/Utils.h" +#include "scheduler/task/SearchTask.h" +#include "scheduler/tasklabel/SpecResLabel.h" +#include "utils/Log.h" + +namespace milvus { +namespace scheduler { + +bool +LargeSQ8HPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { + return false; + } + + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8H) { + return false; + } + + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + + // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu + if (search_job->nq() < 100) { + return false; + } + + std::vector gpus = scheduler::get_gpu_pool(); + std::vector all_free_mem; + for (auto& gpu : gpus) { + auto cache = cache::GpuCacheMgr::GetInstance(gpu); + auto free_mem = cache->CacheCapacity() - cache->CacheUsage(); + all_free_mem.push_back(free_mem); + } + + auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end()); + auto best_index = std::distance(all_free_mem.begin(), max_e); + auto best_device_id = gpus[best_index]; + + ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + if (not res_ptr) { + SERVER_LOG_ERROR << "GpuResource " << best_device_id << " invalid."; + // TODO: throw critical error and exit + return false; + } + + auto label = std::make_shared(std::weak_ptr(res_ptr)); + task->label() = label; + + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.h b/core/src/scheduler/optimizer/LargeSQ8HPass.h new file mode 100644 index 0000000000..49e658002f --- /dev/null +++ b/core/src/scheduler/optimizer/LargeSQ8HPass.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Pass.h" + +namespace milvus { +namespace scheduler { + +class LargeSQ8HPass : public Pass { + public: + LargeSQ8HPass() = default; + + public: + bool + Run(const TaskPtr& task) override; +}; + +using LargeSQ8HPassPtr = std::shared_ptr; + +} // namespace scheduler +} // namespace milvus