From 0032c1344591e84d72f1847ad8735df082c0bfc2 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Thu, 30 Apr 2020 23:22:29 +0800 Subject: [PATCH] Error out when index SQ8H run in CPU mode (#2204) * Error out when index SQ8H run in CPU mode Signed-off-by: yudong.cai * fix error Signed-off-by: yudong.cai * update cmake Signed-off-by: yudong.cai * code opt Signed-off-by: yudong.cai --- CHANGELOG.md | 1 + core/CMakeLists.txt | 6 +++--- core/build.sh | 2 +- core/cmake/DefineOptions.cmake | 2 +- core/src/CMakeLists.txt | 10 ++++++++-- core/src/db/DBImpl.cpp | 5 ++++- core/src/db/engine/ExecutionEngineImpl.cpp | 10 ++++++++++ 7 files changed, 28 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5397bff074..d57b86077e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Please mark all change in change log and use the issue from GitHub ## Bug - \#1705 Limit the insert data batch size +- \#1776 Error out when index SQ8H run in CPU mode - \#1929 Skip MySQL meta schema field width check - \#1946 Fix load index file CPU2GPU fail during searching - \#1997 Index file missed after compact diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 529887c75d..bd79c3e0e1 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -167,9 +167,9 @@ if (MILVUS_WITH_PROMETHEUS) add_compile_definitions("MILVUS_WITH_PROMETHEUS") endif () -message("MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}") -if (MILVUS_ENABLE_PROFILING STREQUAL "ON") - ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING) +message("ENABLE_CPU_PROFILING = ${ENABLE_CPU_PROFILING}") +if (ENABLE_CPU_PROFILING STREQUAL "ON") + ADD_DEFINITIONS(-DENABLE_CPU_PROFILING) endif() if (MILVUS_WITH_FIU) diff --git a/core/build.sh b/core/build.sh index 4374f7bd78..83287d9d4b 100755 --- a/core/build.sh +++ b/core/build.sh @@ -115,7 +115,7 @@ CMAKE_CMD="cmake \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DMILVUS_DB_PATH=${DB_PATH} \ --DMILVUS_ENABLE_PROFILING=${PROFILING} \ +-DENABLE_CPU_PROFILING=${PROFILING} \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ -DFAISS_WITH_MKL=${WITH_MKL} \ -DMILVUS_WITH_PROMETHEUS=${WITH_PROMETHEUS} \ diff --git a/core/cmake/DefineOptions.cmake b/core/cmake/DefineOptions.cmake index e9d5ab1340..1103a9b125 100644 --- a/core/cmake/DefineOptions.cmake +++ b/core/cmake/DefineOptions.cmake @@ -74,7 +74,7 @@ define_option(MILVUS_WITH_MYSQLPP "Build with MySQL++" ON) define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON) -if (MILVUS_ENABLE_PROFILING STREQUAL "ON") +if (ENABLE_CPU_PROFILING STREQUAL "ON") define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON) define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON) endif () diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 7beab7d455..653069af76 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -212,10 +212,16 @@ if (MILVUS_GPU_VERSION) ) endif () -if (ENABLE_CPU_PROFILING OR ENABLE_MEM_PROFILING) +# cannot be enabled together with ENABLE_CPU_PROFILING +if (ENABLE_MEM_PROFILING) + set(third_party_libs ${third_party_libs} + tcmalloc + ) +endif () + +if (ENABLE_CPU_PROFILING) set(third_party_libs ${third_party_libs} gperftools - tcmalloc libunwind ) endif () diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index bbe001354f..46053ce155 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -451,7 +451,10 @@ DBImpl::PreloadCollection(const std::string& collection_id) { fiu_do_on("DBImpl.PreloadCollection.engine_throw_exception", throw std::exception()); std::string msg = "Pre-loaded file: " + file.file_id_ + " size: " + std::to_string(file.file_size_); TimeRecorderAuto rc_1(msg); - engine->Load(true); + status = engine->Load(true); + if (!status.ok()) { + return status; + } size += engine->Size(); if (size > available_size) { diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 0fcd9e2fde..f9e34397ac 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -458,6 +458,16 @@ ExecutionEngineImpl::Load(bool to_cache) { LOG_ENGINE_ERROR_ << msg; return Status(DB_ERROR, msg); } else { + bool gpu_enable = false; +#ifdef MILVUS_GPU_VERSION + server::Config& config = server::Config::GetInstance(); + CONFIG_CHECK(config.GetGpuResourceConfigEnable(gpu_enable)); +#endif + if (!gpu_enable && index_->index_mode() == knowhere::IndexMode::MODE_GPU) { + std::string err_msg = "Index with type " + index_->index_type() + " must be used in GPU mode"; + LOG_ENGINE_ERROR_ << err_msg; + return Status(DB_ERROR, err_msg); + } segment::DeletedDocsPtr deleted_docs_ptr; auto status = segment_reader_ptr->LoadDeletedDocs(deleted_docs_ptr); if (!status.ok()) {