From e24ae02665e742aa76c85b936486c61f0f88374d Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 21 Jul 2019 20:46:24 +0800 Subject: [PATCH 1/3] update faiss version to 1.5.3 and add BUILD_FAISS_WITH_MKL as an option Former-commit-id: 0c26016af4203d4849b8b9b710d005e83e32daea --- cpp/cmake/ThirdPartyPackages.cmake | 35 ++++++++++------------- cpp/src/CMakeLists.txt | 3 +- cpp/thirdparty/versions.txt | 2 +- cpp/unittest/db/CMakeLists.txt | 3 +- cpp/unittest/faiss_wrapper/CMakeLists.txt | 3 +- cpp/unittest/metrics/CMakeLists.txt | 3 +- cpp/unittest/server/CMakeLists.txt | 3 +- 7 files changed, 21 insertions(+), 31 deletions(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 62b55dda81..fed5dece88 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -699,7 +699,9 @@ endmacro() # ---------------------------------------------------------------------- # FAISS -set(BUILD_FAISS_WITH_MKL false) +if(NOT DEFINED BUILD_FAISS_WITH_MKL) + set(BUILD_FAISS_WITH_MKL OFF) +endif() if(EXISTS "/proc/cpuinfo") FILE(READ /proc/cpuinfo PROC_CPUINFO) @@ -708,8 +710,8 @@ if(EXISTS "/proc/cpuinfo") STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}") STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}") - if(${VENDOR_ID} STREQUAL "GenuineIntel") - set(BUILD_FAISS_WITH_MKL true) + if(NOT ${VENDOR_ID} STREQUAL "GenuineIntel") + set(BUILD_FAISS_WITH_MKL OFF) endif() endif() @@ -729,7 +731,7 @@ macro(build_faiss) set(FAISS_CFLAGS ${EP_C_FLAGS}) set(FAISS_CXXFLAGS ${EP_CXX_FLAGS}) - if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") message(STATUS "Build Faiss with MKL") if(NOT DEFINED MKL_LIB_PATH) set(MKL_LIB_PATH "/opt/intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64") @@ -750,10 +752,10 @@ macro(build_faiss) if(${MILVUS_WITH_FAISS_GPU_VERSION} STREQUAL "ON") set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}" - "--with-cuda-arch=\"-gencode=arch=compute_35,code=compute_35\"" - "--with-cuda-arch=\"-gencode=arch=compute_52,code=compute_52\"" - "--with-cuda-arch=\"-gencode=arch=compute_60,code=compute_60\"" - "--with-cuda-arch=\"-gencode=arch=compute_61,code=compute_61\"" + "--with-cuda-arch=\"-gencode=arch=compute_35,code=sm_35\"" + "--with-cuda-arch=\"-gencode=arch=compute_52,code=sm_52\"" + "--with-cuda-arch=\"-gencode=arch=compute_60,code=sm_60\"" + "--with-cuda-arch=\"-gencode=arch=compute_61,code=sm_61\"" ) else() set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda) @@ -767,26 +769,22 @@ macro(build_faiss) "./configure" ${FAISS_CONFIGURE_ARGS} BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} all - COMMAND - cd gpu && ${MAKE} ${MAKE_BUILD_ARGS} + ${MAKE} ${MAKE_BUILD_ARGS} BUILD_IN_SOURCE 1 INSTALL_COMMAND ${MAKE} install - COMMAND - ln -s faiss_ep ../faiss BUILD_BYPRODUCTS ${FAISS_STATIC_LIB}) - if(${BUILD_FAISS_WITH_MKL} STREQUAL "false") + if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) endif() file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}") add_library(faiss SHARED IMPORTED) - if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") set(MKL_LIBS ${MKL_LIB_PATH}/libmkl_intel_ilp64.a ${MKL_LIB_PATH}/libmkl_gnu_thread.a ${MKL_LIB_PATH}/libmkl_core.a) @@ -806,7 +804,7 @@ macro(build_faiss) add_dependencies(faiss faiss_ep) - if(${BUILD_FAISS_WITH_MKL} STREQUAL "false") + if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") add_dependencies(faiss openblas_ep) add_dependencies(faiss lapack_ep) endif() @@ -815,7 +813,7 @@ endmacro() if(MILVUS_WITH_FAISS) - if(${BUILD_FAISS_WITH_MKL} STREQUAL "false") + if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") resolve_dependency(OpenBLAS) get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") @@ -830,10 +828,7 @@ if(MILVUS_WITH_FAISS) resolve_dependency(FAISS) get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM "${FAISS_INCLUDE_DIR}") - include_directories(SYSTEM "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/") - link_directories(SYSTEM ${FAISS_PREFIX}/) link_directories(SYSTEM ${FAISS_PREFIX}/lib/) - link_directories(SYSTEM ${FAISS_PREFIX}/gpu/) endif() # ---------------------------------------------------------------------- diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 64edf5b91e..80603cf673 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -72,7 +72,6 @@ set(third_party_libs sqlite thrift yaml-cpp - libgpufaiss.a faiss prometheus-cpp-push prometheus-cpp-pull @@ -93,7 +92,7 @@ set(third_party_libs if (MEGASEARCH_WITH_ARROW STREQUAL "ON") set(third_party_libs ${third_party_libs} arrow) endif() -if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") +if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") set(third_party_libs ${third_party_libs} ${MKL_LIBS} ${MKL_LIBS}) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index e8c63cad1c..02e20c32cb 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -2,7 +2,7 @@ ARROW_VERSION=zilliz BOOST_VERSION=1.70.0 BZIP2_VERSION=1.0.6 EASYLOGGINGPP_VERSION=v9.96.7 -FAISS_VERSION=7b07685 +FAISS_VERSION=v1.5.3 MKL_VERSION=2019.4.243 GTEST_VERSION=1.8.1 JSONCONS_VERSION=0.126.0 diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 736219952c..0e3531cd33 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -35,7 +35,6 @@ set(db_test_src cuda_add_executable(db_test ${db_test_src}) set(db_libs - libgpufaiss.a faiss cudart cublas @@ -46,7 +45,7 @@ set(db_libs mysqlpp ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") +if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS}) endif() diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index c439250544..f7976fb604 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -24,7 +24,6 @@ set(wrapper_libs stdc++ boost_system_static boost_filesystem_static - libgpufaiss.a faiss cudart cublas @@ -35,7 +34,7 @@ set(wrapper_libs zstd lz4 ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") +if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") set(wrapper_libs ${wrapper_libs} ${MKL_LIBS} ${MKL_LIBS}) endif() diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 418544d0ca..d4ae934d6f 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -49,7 +49,6 @@ set(count_test_src add_executable(metrics_test ${count_test_src} ${require_files} ) target_link_libraries(metrics_test - libgpufaiss.a faiss cudart cublas @@ -64,7 +63,7 @@ target_link_libraries(metrics_test mysqlpp ${unittest_libs} ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") +if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS}) endif() diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 40f5b10128..7a20e6287b 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -33,7 +33,6 @@ cuda_add_executable(server_test set(require_libs stdc++ - libgpufaiss.a faiss cudart cublas @@ -48,7 +47,7 @@ set(require_libs pthread ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") +if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS}) endif() From 42b448d9abbdb129898e87ccd1caf8ea5fcd43a0 Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 21 Jul 2019 20:47:28 +0800 Subject: [PATCH 2/3] update faiss version to 1.5.3 and add BUILD_FAISS_WITH_MKL as an option Former-commit-id: c3efd4e0515a5145779a2d0fde552520d04e6d01 --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 56c5c5faa8..77cbe055e3 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -35,6 +35,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-248 - Support AddVector/SearchVector profiling - MS-256 - Add more cache config - MS-260 - Refine log +- MS-261 - update faiss version to 1.5.3 and add BUILD_FAISS_WITH_MKL as an option ## New Feature - MS-180 - Add new mem manager From 241320fb5027736fb03e84bae8fbd6be52ffd0a2 Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 21 Jul 2019 21:01:53 +0800 Subject: [PATCH 3/3] update faiss version to 1.5.3 and add BUILD_FAISS_WITH_MKL as an option Former-commit-id: 2156052d9cf246e72168c4490d0b034ad6a00e47 --- cpp/build.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cpp/build.sh b/cpp/build.sh index 52ac97d22a..edfe9305be 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -8,8 +8,9 @@ MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" DB_PATH="/opt/milvus" PROFILING="OFF" +BUILD_FAISS_WITH_MKL="OFF" -while getopts "p:d:t:uhlrcg" arg +while getopts "p:d:t:uhlrcgm" arg do case $arg in t) @@ -40,6 +41,9 @@ do g) PROFILING="ON" ;; + m) + BUILD_FAISS_WITH_MKL="ON" + ;; h) # help echo " @@ -52,9 +56,10 @@ parameter: -r: remove previous build directory(default: OFF) -c: code coverage(default: OFF) -g: profiling(default: OFF) +-m: build faiss with MKL(default: OFF) usage: -./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] +./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] [-m] " exit 0 ;; @@ -83,6 +88,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DMILVUS_DB_PATH=${DB_PATH} \ -DMILVUS_ENABLE_PROFILING=${PROFILING} \ + -DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \ $@ ../" echo ${CMAKE_CMD}