From 12f87049a09712b30d73646adb90457d7de830ac Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Wed, 13 Nov 2019 15:46:45 +0800 Subject: [PATCH 1/5] update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be8bc36b3b..fefd4e54c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,12 @@ Please mark all change in change log and use the ticket from JIRA. - \#12 - Pure CPU version for Milvus - \#77 - Support table partition - \#226 - Experimental shards middleware for Milvus -- \#314 - add Find FAISS in CMake ## Improvement - \#275 - Rename C++ SDK IndexType - \#284 - Change C++ SDK to shared library - \#260 - C++ SDK README +- \#314 - add Find FAISS in CMake ## Task From 1fa4c2531a2762130b9707bace08805c4601a07f Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Wed, 13 Nov 2019 16:01:03 +0800 Subject: [PATCH 2/5] remove FAISS_URL --- .../index/cmake/ThirdPartyPackagesCore.cmake | 22 +++---------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/core/src/index/cmake/ThirdPartyPackagesCore.cmake index 8563e37e87..854b295b1a 100644 --- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -225,28 +225,12 @@ foreach (_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) set(${_LIB_NAME} "${_LIB_VERSION}") endforeach () -if (CUSTOMIZATION) - execute_process(COMMAND wget -q --method HEAD ${FAISS_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote cache file ${FAISS_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - MESSAGE(FATAL_ERROR "Can't access to ${FAISS_URL}") - else () - set(FAISS_SOURCE_URL ${FAISS_URL}) - # set(FAISS_MD5 "a589663865a8558205533c8ac414278c") - # set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0 - # set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0 - # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1 - # set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1 - # set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1 - # set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0 - # set(FAISS_MD5 "bb30722c22390ce5f6759ccb216c1b2a") # commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0 - set(FAISS_MD5 "2293cdb209c3718e3b19f3edae8b32b3") # commit-id a13c1205dc52977a9ad3b33a14efa958604a8bff branch-0.3.0 - endif () +if (DEFINED ENV{FAISS_SOURCE_URL}) + set(FAISS_SOURCE_URL "$ENV{FAISS_SOURCE_URL}") else () - set(FAISS_SOURCE_URL "https://github.com/JinHai-CN/faiss/archive/1.6.0.tar.gz") + set(FAISS_SOURCE_URL "https://github.com/milvus-io/faiss/archive/${FAISS_VERSION}.tar.gz") set(FAISS_MD5 "b02c1a53234f5acc9bea1b0c55524f50") endif () -message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}") if (DEFINED ENV{KNOWHERE_ARROW_URL}) set(ARROW_SOURCE_URL "$ENV{KNOWHERE_ARROW_URL}") From de1aa04bd0cf5666d325cb0ff08a61689bb6c8f9 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Wed, 13 Nov 2019 16:11:31 +0800 Subject: [PATCH 3/5] update build.sh --- core/build.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/core/build.sh b/core/build.sh index 5abdaf175a..9b690a0261 100755 --- a/core/build.sh +++ b/core/build.sh @@ -14,16 +14,10 @@ CUSTOMIZATION="OFF" # default use ori faiss CUDA_COMPILER=/usr/local/cuda/bin/nvcc GPU_VERSION="OFF" #defaults to CPU version WITH_MKL="OFF" +FAISS_ROOT="" +FAISS_SOURCE="BUNDLED" -CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}" -wget -q --method HEAD ${CUSTOMIZED_FAISS_URL} -if [ $? -eq 0 ]; then - CUSTOMIZATION="ON" -else - CUSTOMIZATION="OFF" -fi - -while getopts "p:d:t:ulrcgjhxzm" arg +while getopts "p:d:t:f:ulrcgjhxzm" arg do case $arg in p) @@ -35,6 +29,10 @@ do t) BUILD_TYPE=$OPTARG # BUILD_TYPE ;; + f) + FAISS_ROOT=$OPTARG + FAISS_SOURCE="AUTO" + ;; u) echo "Build and run unittest cases" ; BUILD_UNITTEST="ON"; @@ -73,6 +71,7 @@ parameter: -p: install prefix(default: $(pwd)/milvus) -d: db data path(default: /tmp/milvus) -t: build type(default: Debug) +-f: faiss root path(default: empty) -u: building unit test options(default: OFF) -l: run cpplint, clang-format and clang-tidy(default: OFF) -r: remove previous build directory(default: OFF) @@ -84,7 +83,7 @@ parameter: -h: help usage: -./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h] +./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h] " exit 0 ;; @@ -109,13 +108,14 @@ CMAKE_CMD="cmake \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ +-DFAISS_ROOT=${FAISS_ROOT} \ +-DFAISS_SOURCE=${FAISS_SOURCE} \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DMILVUS_DB_PATH=${DB_PATH} \ -DMILVUS_ENABLE_PROFILING=${PROFILING} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ -DCUSTOMIZATION=${CUSTOMIZATION} \ --DFAISS_URL=${CUSTOMIZED_FAISS_URL} \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ -DFAISS_WITH_MKL=${WITH_MKL} \ ../" From 5f4866616d0f6d33fa6a9e596a5979a1f1ff4cd7 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Wed, 13 Nov 2019 16:36:40 +0800 Subject: [PATCH 4/5] update build.sh in ci --- ci/scripts/build.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 444537fe6b..27962ccb54 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -15,6 +15,7 @@ CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build" BUILD_TYPE="Debug" BUILD_UNITTEST="OFF" INSTALL_PREFIX="/opt/milvus" +FAISS_ROOT="" BUILD_COVERAGE="OFF" USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" @@ -22,7 +23,7 @@ GPU_VERSION="OFF" WITH_MKL="OFF" CUDA_COMPILER=/usr/local/cuda/bin/nvcc -while getopts "o:t:b:gulcjmh" arg +while getopts "o:t:b:f:gulcjmh" arg do case $arg in o) @@ -34,6 +35,9 @@ do b) CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR ;; + f) + FAISS_ROOT=$OPTARG # FAISS ROOT PATH + ;; g) GPU_VERSION="ON"; ;; @@ -60,6 +64,7 @@ parameter: -o: install prefix(default: /opt/milvus) -t: build type(default: Debug) -b: core code build directory +-f: faiss root path -g: gpu version -u: building unit test options(default: OFF) -l: run cpplint, clang-format and clang-tidy(default: OFF) @@ -69,7 +74,7 @@ parameter: -h: help usage: -./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} [-u] [-l] [-c] [-j] [-m] [-h] +./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-u] [-l] [-c] [-j] [-m] [-h] " exit 0 ;; @@ -94,8 +99,10 @@ CMAKE_CMD="cmake \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ --DBUILD_FAISS_WITH_MKL=${WITH_MKL} \ +-DFAISS_ROOT=${FAISS_ROOT} \ +-DFAISS_WITH_MKL=${WITH_MKL} \ -DArrow_SOURCE=AUTO \ +-DFAISS_SOURCE=AUTO \ ${MILVUS_CORE_DIR}" echo ${CMAKE_CMD} ${CMAKE_CMD} From ff7452a21fc9f8ef80aa0162c1d9f250d8aec6c4 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Wed, 13 Nov 2019 16:53:23 +0800 Subject: [PATCH 5/5] fix faiss source url --- core/src/index/cmake/ThirdPartyPackagesCore.cmake | 2 +- core/src/index/thirdparty/versions.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/core/src/index/cmake/ThirdPartyPackagesCore.cmake index 854b295b1a..c046bc3a56 100644 --- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -228,7 +228,7 @@ endforeach () if (DEFINED ENV{FAISS_SOURCE_URL}) set(FAISS_SOURCE_URL "$ENV{FAISS_SOURCE_URL}") else () - set(FAISS_SOURCE_URL "https://github.com/milvus-io/faiss/archive/${FAISS_VERSION}.tar.gz") + set(FAISS_SOURCE_URL "https://github.com/JinHai-CN/faiss/archive/${FAISS_VERSION}.tar.gz") set(FAISS_MD5 "b02c1a53234f5acc9bea1b0c55524f50") endif () diff --git a/core/src/index/thirdparty/versions.txt b/core/src/index/thirdparty/versions.txt index 380c9dedad..efcef26fa9 100644 --- a/core/src/index/thirdparty/versions.txt +++ b/core/src/index/thirdparty/versions.txt @@ -3,5 +3,5 @@ BOOST_VERSION=1.70.0 GTEST_VERSION=1.8.1 LAPACK_VERSION=v3.8.0 OPENBLAS_VERSION=v0.3.6 -FAISS_VERSION=branch-0.3.0 +FAISS_VERSION=1.6.0 MKL_VERSION=2019.5.281