From 054a7b0d20528e5b54503f11b3252f868e846424 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 17 Jun 2019 10:28:26 +0800 Subject: [PATCH 1/4] achangelog Former-commit-id: 0266c20614775fd7dfd3b91fa2cbc50807765afc --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 4134528d64..2231ab6675 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -11,6 +11,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-82 - Update server startup welcome message - MS-83 - Update vecwise to Milvus - MS-77 - Performance issue of post-search action +- MS-22 - Enhancement for MemVector size control ## New Feature From 374b30794cc79c673bb6687020205a87c64a8542 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 17 Jun 2019 19:25:48 +0800 Subject: [PATCH 2/4] MS-92 - Unify behavior of debug and release build Former-commit-id: cf2ab198bf28c9bf26cd0acb239c203626f7592f --- cpp/CHANGELOG.md | 1 + cpp/CMakeLists.txt | 6 ++---- cpp/README.md | 1 - cpp/build.sh | 10 +++++----- cpp/src/db/DBMetaImpl.cpp | 2 +- cpp/src/wrapper/IndexBuilder.cpp | 3 +++ cpp/unittest/db/CMakeLists.txt | 1 + cpp/unittest/db/db_tests.cpp | 3 +++ cpp/unittest/faiss_wrapper/CMakeLists.txt | 1 + cpp/unittest/metrics/CMakeLists.txt | 2 +- cpp/unittest/server/CMakeLists.txt | 1 + 11 files changed, 19 insertions(+), 12 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 63d57e4beb..5f92adafcc 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -13,6 +13,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-83 - Update vecwise to Milvus - MS-77 - Performance issue of post-search action - MS-22 - Enhancement for MemVector size control +- MS-92 - Unify behavior of debug and release build ## New Feature diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 7973225cee..ce854fe527 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -71,9 +71,7 @@ set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") message("CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}") message("CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}") -if (GPU_VERSION STREQUAL "ON") - add_definitions("-DGPU_VERSION") -endif () +add_definitions("-DGPU_VERSION") set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED on) @@ -92,7 +90,7 @@ endif() if(CMAKE_BUILD_TYPE STREQUAL "Release") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") - if (GPU_VERSION STREQUAL "ON") + if (CMAKE_LICENSE_CHECK STREQUAL "ON") set(ENABLE_LICENSE "ON") add_definitions("-DENABLE_LICENSE") endif () diff --git a/cpp/README.md b/cpp/README.md index 7e739badcc..f93f7965db 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -15,7 +15,6 @@ cmake_build/src/libmilvus_engine.a is the static library cd [sourcecode path]/cpp ./build.sh -t Debug ./build.sh -t Release - ./build.sh -g # Build GPU version If you encounter the following error when building: `protocol https not supported or disabled in libcurl` diff --git a/cpp/build.sh b/cpp/build.sh index 79a128e45e..0603ae5623 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -2,7 +2,7 @@ BUILD_TYPE="Debug" BUILD_UNITTEST="off" -BUILD_GPU="OFF" +LICENSE_CHECK="OFF" INSTALL_PREFIX=$(pwd)/milvus MAKE_CLEAN="OFF" @@ -19,8 +19,8 @@ do p) INSTALL_PREFIX=$OPTARG ;; - g) - BUILD_GPU="ON" + l) + LICENSE_CHECK="ON" ;; r) if [[ -d cmake_build ]]; then @@ -35,7 +35,7 @@ parameter: -t: build type -u: building unit test options -p: install prefix --g: build GPU version +-l: build license version -r: remove previous build directory usage: @@ -64,7 +64,7 @@ if [[ ${MAKE_CLEAN} = "ON" ]]; then -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ - -DGPU_VERSION=${BUILD_GPU} \ + -DCMAKE_LICENSE_CHECK=${LICENSE_CHECK} \ $@ ../" echo ${CMAKE_CMD} diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index ceaf0f8c68..7305060c0a 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -318,7 +318,7 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) { file_schema.updated_time_ = file_schema.created_on_; file_schema.engine_type_ = table_schema.engine_type_; GetTableFilePath(file_schema); - + ENGINE_LOG_DEBUG << "CreateTableFile " << file_schema.file_id_; { try { server::Metrics::GetInstance().MetaAccessTotalIncrement(); diff --git a/cpp/src/wrapper/IndexBuilder.cpp b/cpp/src/wrapper/IndexBuilder.cpp index 43cbb73235..d4429c381a 100644 --- a/cpp/src/wrapper/IndexBuilder.cpp +++ b/cpp/src/wrapper/IndexBuilder.cpp @@ -69,6 +69,7 @@ Index_ptr IndexBuilder::build_all(const long &nb, std::shared_ptr host_index = nullptr; #ifdef GPU_VERSION { + LOG(DEBUG) << "Build index by GPU"; // TODO: list support index-type. faiss::Index *ori_index = faiss::index_factory(opd_->d, opd_->get_index_type(nb).c_str()); @@ -88,6 +89,7 @@ Index_ptr IndexBuilder::build_all(const long &nb, } #else { + LOG(DEBUG) << "Build index by CPU"; faiss::Index *index = faiss::index_factory(opd_->d, opd_->get_index_type(nb).c_str()); if (!index->is_trained) { nt == 0 || xt == nullptr ? index->train(nb, xb) @@ -113,6 +115,7 @@ Index_ptr BgCpuBuilder::build_all(const long &nb, const float *xb, const long *i std::shared_ptr index = nullptr; index.reset(faiss::index_factory(opd_->d, opd_->get_index_type(nb).c_str())); + LOG(DEBUG) << "Build index by CPU"; { std::lock_guard lk(cpu_resource); if (!index->is_trained) { diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index ecfeb1e92c..b8985067e9 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -35,6 +35,7 @@ set(db_test_src cuda_add_executable(db_test ${db_test_src}) set(db_libs + gpufaiss faiss cudart cublas diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 1538dd486a..00b6c7a9c1 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -69,6 +69,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { engine::meta::TableSchema group_info; group_info.dimension_ = group_dim; group_info.table_id_ = group_name; + group_info.engine_type_ = (int)engine::EngineType::FAISS_IVFFLAT; engine::Status stat = db_->CreateTable(group_info); engine::meta::TableSchema group_info_get; @@ -116,6 +117,7 @@ TEST_F(DBTest, DB_TEST) { engine::meta::TableSchema group_info; group_info.dimension_ = group_dim; group_info.table_id_ = group_name; + group_info.engine_type_ = (int)engine::EngineType::FAISS_IVFFLAT; engine::Status stat = db_->CreateTable(group_info); engine::meta::TableSchema group_info_get; @@ -202,6 +204,7 @@ TEST_F(DBTest, SEARCH_TEST) { engine::meta::TableSchema group_info; group_info.dimension_ = group_dim; group_info.table_id_ = group_name; + group_info.engine_type_ = (int)engine::EngineType::FAISS_IVFFLAT; engine::Status stat = db_->CreateTable(group_info); engine::meta::TableSchema group_info_get; diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index 49fdd7d605..76c8ca9777 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -32,6 +32,7 @@ set(wrapper_libs stdc++ boost_system boost_filesystem + gpufaiss faiss cudart cublas diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index f24538fad4..8a34d9c4d7 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -60,7 +60,7 @@ set(count_test_src add_executable(metrics_test ${count_test_src} ${require_files} ) target_link_libraries(metrics_test - + gpufaiss faiss cudart cublas diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 8bc2e41bee..3257804abf 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -33,6 +33,7 @@ cuda_add_executable(server_test set(require_libs stdc++ + gpufaiss faiss cudart cublas From 4f32022df3b0f6ca7277ec59a0524d498e621269 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 17 Jun 2019 19:29:51 +0800 Subject: [PATCH 3/4] MS-92 - Unify behavior of debug and release build Former-commit-id: 873447e6e43b94657cb86a27e01cdedc9652aa7b --- cpp/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/README.md b/cpp/README.md index f93f7965db..0c7706df23 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -15,6 +15,7 @@ cmake_build/src/libmilvus_engine.a is the static library cd [sourcecode path]/cpp ./build.sh -t Debug ./build.sh -t Release + ./build.sh -l -t Release # Build license version(only available for Release) If you encounter the following error when building: `protocol https not supported or disabled in libcurl` From fb23d1ba8d8c1019cc14d9cfc4f158bca57aa433 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 17 Jun 2019 19:32:28 +0800 Subject: [PATCH 4/4] MS-92 - Unify behavior of debug and release build Former-commit-id: 0026e992f51578071ea57597c4389d07bc7bc7cb --- cpp/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/build.sh b/cpp/build.sh index 0603ae5623..3af7b31bfb 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -6,7 +6,7 @@ LICENSE_CHECK="OFF" INSTALL_PREFIX=$(pwd)/milvus MAKE_CLEAN="OFF" -while getopts "p:t:uhgr" arg +while getopts "p:t:uhlr" arg do case $arg in t)