Merge branch 'branch-0.3.0' into 'branch-0.3.0'

MS-92 - Unify behavior of debug and release build

See merge request megasearch/vecwise_engine!99

Former-commit-id: 34c8a1d9bd6373f1d97a4b010de7c7f371eb97ac

cpp/CHANGELOG.md

@@ -13,6 +13,8 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-82 - Update server startup welcome message
 - MS-83 - Update vecwise to Milvus
 - MS-77 - Performance issue of post-search action
+- MS-22 - Enhancement for MemVector size control 
+- MS-92 - Unify behavior of debug and release build
 
 ## New Feature
 

cpp/CMakeLists.txt

@@ -71,9 +71,7 @@ set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
 message("CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}")
 message("CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}")
 
-if (GPU_VERSION STREQUAL "ON")
-    add_definitions("-DGPU_VERSION")
-endif ()
+add_definitions("-DGPU_VERSION")
 
 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_STANDARD_REQUIRED on)
@@ -92,7 +90,7 @@ endif()
 
 if(CMAKE_BUILD_TYPE STREQUAL "Release")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
-     if (GPU_VERSION STREQUAL "ON")
+     if (CMAKE_LICENSE_CHECK STREQUAL "ON")
          set(ENABLE_LICENSE "ON")
          add_definitions("-DENABLE_LICENSE")
      endif ()

cpp/README.md

@@ -15,7 +15,7 @@ cmake_build/src/libmilvus_engine.a is the static library
     cd [sourcecode path]/cpp
     ./build.sh -t Debug
     ./build.sh -t Release
-    ./build.sh -g # Build GPU version
+    ./build.sh -l -t Release # Build license version(only available for Release)
 
 If you encounter the following error when building:
 `protocol https not supported or disabled in libcurl`

cpp/build.sh

@@ -2,11 +2,11 @@
 
 BUILD_TYPE="Debug"
 BUILD_UNITTEST="off"
-BUILD_GPU="OFF"
+LICENSE_CHECK="OFF"
 INSTALL_PREFIX=$(pwd)/milvus
 MAKE_CLEAN="OFF"
 
-while getopts "p:t:uhgr" arg
+while getopts "p:t:uhlr" arg
 do
         case $arg in
              t)
@@ -19,8 +19,8 @@ do
              p)
                 INSTALL_PREFIX=$OPTARG
                 ;;
-             g)
-                BUILD_GPU="ON"
+             l)
+                LICENSE_CHECK="ON"
                 ;;
              r)
                 if [[ -d cmake_build ]]; then
@@ -35,7 +35,7 @@ parameter:
 -t: build type
 -u: building unit test options
 -p: install prefix
--g: build GPU version
+-l: build license version
 -r: remove previous build directory
 
 usage:
@@ -64,7 +64,7 @@ if [[ ${MAKE_CLEAN} = "ON" ]]; then
     -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
     -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
     -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
-    -DGPU_VERSION=${BUILD_GPU} \
+    -DCMAKE_LICENSE_CHECK=${LICENSE_CHECK} \
     $@ ../"
     echo ${CMAKE_CMD}
 

cpp/src/db/DBMetaImpl.cpp

@@ -318,7 +318,7 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
     file_schema.updated_time_ = file_schema.created_on_;
     file_schema.engine_type_ = table_schema.engine_type_;
     GetTableFilePath(file_schema);
-
+    ENGINE_LOG_DEBUG << "CreateTableFile " << file_schema.file_id_;
     {
         try {
             server::Metrics::GetInstance().MetaAccessTotalIncrement();

cpp/src/wrapper/IndexBuilder.cpp

@@ -69,6 +69,7 @@ Index_ptr IndexBuilder::build_all(const long &nb,
     std::shared_ptr<faiss::Index> host_index = nullptr;
 #ifdef GPU_VERSION
     {
+        LOG(DEBUG) << "Build index by GPU";
         // TODO: list support index-type.
         faiss::Index *ori_index = faiss::index_factory(opd_->d, opd_->get_index_type(nb).c_str());
 
@@ -88,6 +89,7 @@ Index_ptr IndexBuilder::build_all(const long &nb,
     }
 #else
     {
+        LOG(DEBUG) << "Build index by CPU";
         faiss::Index *index = faiss::index_factory(opd_->d, opd_->get_index_type(nb).c_str());
         if (!index->is_trained) {
             nt == 0 || xt == nullptr ? index->train(nb, xb)
@@ -113,6 +115,7 @@ Index_ptr BgCpuBuilder::build_all(const long &nb, const float *xb, const long *i
     std::shared_ptr<faiss::Index> index = nullptr;
     index.reset(faiss::index_factory(opd_->d, opd_->get_index_type(nb).c_str()));
 
+    LOG(DEBUG) << "Build index by CPU";
     {
         std::lock_guard<std::mutex> lk(cpu_resource);
         if (!index->is_trained) {

cpp/unittest/db/CMakeLists.txt

@@ -35,6 +35,7 @@ set(db_test_src
 cuda_add_executable(db_test ${db_test_src})
 
 set(db_libs
+        gpufaiss
         faiss
         cudart
         cublas

cpp/unittest/db/db_tests.cpp

@@ -69,6 +69,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
     engine::meta::TableSchema group_info;
     group_info.dimension_ = group_dim;
     group_info.table_id_ = group_name;
+    group_info.engine_type_ = (int)engine::EngineType::FAISS_IVFFLAT;
     engine::Status stat = db_->CreateTable(group_info);
 
     engine::meta::TableSchema group_info_get;
@@ -116,6 +117,7 @@ TEST_F(DBTest, DB_TEST) {
     engine::meta::TableSchema group_info;
     group_info.dimension_ = group_dim;
     group_info.table_id_ = group_name;
+    group_info.engine_type_ = (int)engine::EngineType::FAISS_IVFFLAT;
     engine::Status stat = db_->CreateTable(group_info);
 
     engine::meta::TableSchema group_info_get;
@@ -202,6 +204,7 @@ TEST_F(DBTest, SEARCH_TEST) {
     engine::meta::TableSchema group_info;
     group_info.dimension_ = group_dim;
     group_info.table_id_ = group_name;
+    group_info.engine_type_ = (int)engine::EngineType::FAISS_IVFFLAT;
     engine::Status stat = db_->CreateTable(group_info);
 
     engine::meta::TableSchema group_info_get;

cpp/unittest/faiss_wrapper/CMakeLists.txt

@@ -32,6 +32,7 @@ set(wrapper_libs
         stdc++
         boost_system
         boost_filesystem
+        gpufaiss
         faiss
         cudart
         cublas

cpp/unittest/metrics/CMakeLists.txt

@@ -60,7 +60,7 @@ set(count_test_src
 add_executable(metrics_test ${count_test_src} ${require_files} )
 
 target_link_libraries(metrics_test
-
+        gpufaiss
         faiss
         cudart
         cublas

cpp/unittest/server/CMakeLists.txt

@@ -33,6 +33,7 @@ cuda_add_executable(server_test
 
 set(require_libs
         stdc++
+        gpufaiss
         faiss
         cudart
         cublas