admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #317 from youny626/0.6.0

Browse Source 
add Find FAISS in CMake
This commit is contained in:
Jin Hai 2019-11-13 18:19:54 +08:00 committed by GitHub
commit bea4ddc143
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 130 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -16,6 +16,7 @@ Please mark all change in change log and use the ticket from JIRA.
- \#275 - Rename C++ SDK IndexType
- \#284 - Change C++ SDK to shared library
- \#260 - C++ SDK README
- \#314 - add Find FAISS in CMake
## Task

13
ci/scripts/build.sh
View File

@ -15,6 +15,7 @@ CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build"
BUILD_TYPE="Debug"
BUILD_UNITTEST="OFF"
INSTALL_PREFIX="/opt/milvus"
FAISS_ROOT=""
BUILD_COVERAGE="OFF"
USE_JFROG_CACHE="OFF"
RUN_CPPLINT="OFF"
@ -22,7 +23,7 @@ GPU_VERSION="OFF"
WITH_MKL="OFF"
CUDA_COMPILER=/usr/local/cuda/bin/nvcc
while getopts "o:t:b:gulcjmh" arg
while getopts "o:t:b:f:gulcjmh" arg
do
case $arg in
o)
@ -34,6 +35,9 @@ do
b)
CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR
;;
f)
FAISS_ROOT=$OPTARG # FAISS ROOT PATH
;;
g)
GPU_VERSION="ON";
;;
@ -60,6 +64,7 @@ parameter:
-o: install prefix(default: /opt/milvus)
-t: build type(default: Debug)
-b: core code build directory
-f: faiss root path
-g: gpu version
-u: building unit test options(default: OFF)
-l: run cpplint, clang-format and clang-tidy(default: OFF)
@ -69,7 +74,7 @@ parameter:
-h: help
usage:
./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} [-u] [-l] [-c] [-j] [-m] [-h]
./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-u] [-l] [-c] [-j] [-m] [-h]
"
exit 0
;;
@ -94,8 +99,10 @@ CMAKE_CMD="cmake \
-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
-DBUILD_COVERAGE=${BUILD_COVERAGE} \
-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
-DBUILD_FAISS_WITH_MKL=${WITH_MKL} \
-DFAISS_ROOT=${FAISS_ROOT} \
-DFAISS_WITH_MKL=${WITH_MKL} \
-DArrow_SOURCE=AUTO \
-DFAISS_SOURCE=AUTO \
${MILVUS_CORE_DIR}"
echo ${CMAKE_CMD}
${CMAKE_CMD}

24
core/build.sh
View File

@ -14,16 +14,10 @@ CUSTOMIZATION="OFF" # default use ori faiss
CUDA_COMPILER=/usr/local/cuda/bin/nvcc
GPU_VERSION="OFF" #defaults to CPU version
WITH_MKL="OFF"
FAISS_ROOT=""
FAISS_SOURCE="BUNDLED"
CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}"
wget -q --method HEAD ${CUSTOMIZED_FAISS_URL}
if [ $? -eq 0 ]; then
CUSTOMIZATION="ON"
else
CUSTOMIZATION="OFF"
fi
while getopts "p:d:t:ulrcgjhxzm" arg
while getopts "p:d:t:f:ulrcgjhxzm" arg
do
case $arg in
p)
@ -35,6 +29,10 @@ do
t)
BUILD_TYPE=$OPTARG # BUILD_TYPE
;;
f)
FAISS_ROOT=$OPTARG
FAISS_SOURCE="AUTO"
;;
u)
echo "Build and run unittest cases" ;
BUILD_UNITTEST="ON";
@ -73,6 +71,7 @@ parameter:
-p: install prefix(default: $(pwd)/milvus)
-d: db data path(default: /tmp/milvus)
-t: build type(default: Debug)
-f: faiss root path(default: empty)
-u: building unit test options(default: OFF)
-l: run cpplint, clang-format and clang-tidy(default: OFF)
-r: remove previous build directory(default: OFF)
@ -84,7 +83,7 @@ parameter:
-h: help
usage:
./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h]
./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h]
"
exit 0
;;
@ -109,15 +108,16 @@ CMAKE_CMD="cmake \
-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DFAISS_ROOT=${FAISS_ROOT} \
-DFAISS_SOURCE=${FAISS_SOURCE} \
-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
-DBUILD_COVERAGE=${BUILD_COVERAGE} \
-DMILVUS_DB_PATH=${DB_PATH} \
-DMILVUS_ENABLE_PROFILING=${PROFILING} \
-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
-DCUSTOMIZATION=${CUSTOMIZATION} \
-DFAISS_URL=${CUSTOMIZED_FAISS_URL} \
-DMILVUS_GPU_VERSION=${GPU_VERSION} \
-DBUILD_FAISS_WITH_MKL=${WITH_MKL} \
-DFAISS_WITH_MKL=${WITH_MKL} \
../"
echo ${CMAKE_CMD}
${CMAKE_CMD}

2
core/src/CMakeLists.txt
View File

@ -24,7 +24,7 @@ include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status)
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus)
add_subdirectory(index)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
add_compile_definitions("WITH_MKL")
endif ()

6
core/src/index/build.sh
View File

@ -5,7 +5,7 @@ BUILD_UNITTEST="OFF"
INSTALL_PREFIX=$(pwd)/cmake_build
MAKE_CLEAN="OFF"
PROFILING="OFF"
BUILD_FAISS_WITH_MKL="OFF"
FAISS_WITH_MKL="OFF"
USE_JFROG_CACHE="OFF"
while getopts "p:d:t:uhrcgmj" arg
@ -31,7 +31,7 @@ do
PROFILING="ON"
;;
m)
BUILD_FAISS_WITH_MKL="ON"
FAISS_WITH_MKL="ON"
;;
j)
USE_JFROG_CACHE="ON"
@ -74,7 +74,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
-DMILVUS_ENABLE_PROFILING=${PROFILING} \
-DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \
-DFAISS_WITH_MKL=${FAISS_WITH_MKL} \
-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
../"
echo ${CMAKE_CMD}

2
core/src/index/cmake/DefineOptionsCore.cmake
View File

@ -79,7 +79,7 @@ define_option(KNOWHERE_WITH_FAISS "Build with FAISS library" ON)
define_option(KNOWHERE_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON)
define_option(BUILD_FAISS_WITH_MKL "Build FAISS with MKL" OFF)
define_option(FAISS_WITH_MKL "Build FAISS with MKL" OFF)
#----------------------------------------------------------------------
set_option_category("Test and benchmark")

44
core/src/index/cmake/FindFAISS.cmake Normal file
View File

@ -0,0 +1,44 @@
set(FAISS_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX})
# First, find via if specified FAISS_ROOT
if (FAISS_ROOT)
find_library(FAISS_STATIC_LIB
NAMES ${FAISS_STATIC_LIB_NAME}
PATHS ${FAISS_ROOT}
PATH_SUFFIXES "lib"
NO_DEFAULT_PATH
)
find_path(FAISS_INCLUDE_DIR
NAMES "faiss/Index.h"
PATHS ${FAISS_ROOT}
NO_DEFAULT_PATH
PATH_SUFFIXES "include"
)
endif ()
find_package_handle_standard_args(FAISS REQUIRED_VARS FAISS_STATIC_LIB FAISS_INCLUDE_DIR)
if (FAISS_FOUND)
if (NOT TARGET faiss)
add_library(faiss STATIC IMPORTED)
set_target_properties(
faiss
PROPERTIES
IMPORTED_LOCATION "${FAISS_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}"
)
if (FAISS_WITH_MKL)
set_target_properties(
faiss
PROPERTIES
INTERFACE_LINK_LIBRARIES "${MKL_LIBS}")
else ()
set_target_properties(
faiss
PROPERTIES
INTERFACE_LINK_LIBRARIES "openblas;lapack")
endif ()
endif ()
endif ()

116
core/src/index/cmake/ThirdPartyPackagesCore.cmake
View File

@ -21,6 +21,7 @@ set(KNOWHERE_THIRDPARTY_DEPENDENCIES
GTest
LAPACK
OpenBLAS
MKL
)
message(STATUS "Using ${KNOWHERE_DEPENDENCY_SOURCE} approach to find dependencies")
@ -43,6 +44,8 @@ macro(build_dependency DEPENDENCY_NAME)
build_openblas()
elseif ("${DEPENDENCY_NAME}" STREQUAL "FAISS")
build_faiss()
elseif ("${DEPENDENCY_NAME}" STREQUAL "MKL")
build_mkl()
else ()
message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}")
endif ()
@ -51,9 +54,9 @@ endmacro()
macro(resolve_dependency DEPENDENCY_NAME)
if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
find_package(${DEPENDENCY_NAME} MODULE)
if(NOT ${${DEPENDENCY_NAME}_FOUND})
build_dependency(${DEPENDENCY_NAME})
endif()
if (NOT ${${DEPENDENCY_NAME}_FOUND})
build_dependency(${DEPENDENCY_NAME})
endif ()
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
build_dependency(${DEPENDENCY_NAME})
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")
@ -222,28 +225,12 @@ foreach (_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
set(${_LIB_NAME} "${_LIB_VERSION}")
endforeach ()
if (CUSTOMIZATION)
execute_process(COMMAND wget -q --method HEAD ${FAISS_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote cache file ${FAISS_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
MESSAGE(FATAL_ERROR "Can't access to ${FAISS_URL}")
else ()
set(FAISS_SOURCE_URL ${FAISS_URL})
# set(FAISS_MD5 "a589663865a8558205533c8ac414278c")
# set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0
# set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0
# set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
# set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1
# set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1
# set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
# set(FAISS_MD5 "bb30722c22390ce5f6759ccb216c1b2a") # commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0
set(FAISS_MD5 "2293cdb209c3718e3b19f3edae8b32b3") # commit-id a13c1205dc52977a9ad3b33a14efa958604a8bff branch-0.3.0
endif()
else()
set(FAISS_SOURCE_URL "https://github.com/JinHai-CN/faiss/archive/1.6.0.tar.gz")
if (DEFINED ENV{FAISS_SOURCE_URL})
set(FAISS_SOURCE_URL "$ENV{FAISS_SOURCE_URL}")
else ()
set(FAISS_SOURCE_URL "https://github.com/JinHai-CN/faiss/archive/${FAISS_VERSION}.tar.gz")
set(FAISS_MD5 "b02c1a53234f5acc9bea1b0c55524f50")
endif()
message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}")
endif ()
if (DEFINED ENV{KNOWHERE_ARROW_URL})
set(ARROW_SOURCE_URL "$ENV{KNOWHERE_ARROW_URL}")
@ -673,28 +660,46 @@ if (KNOWHERE_BUILD_TESTS AND NOT TARGET googletest_ep)
include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
endif ()
# ----------------------------------------------------------------------
# MKL
macro(build_mkl)
if (FAISS_WITH_MKL)
if (EXISTS "/proc/cpuinfo")
FILE(READ /proc/cpuinfo PROC_CPUINFO)
SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n")
STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}")
STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}")
if (NOT ${VENDOR_ID} STREQUAL "GenuineIntel")
set(FAISS_WITH_MKL OFF)
endif ()
endif ()
find_path(MKL_LIB_PATH
NAMES "libmkl_intel_ilp64.a" "libmkl_gnu_thread.a" "libmkl_core.a"
PATH_SUFFIXES "intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64/")
if (${MKL_LIB_PATH} STREQUAL "MKL_LIB_PATH-NOTFOUND")
message(FATAL_ERROR "Could not find MKL libraries")
endif ()
message(STATUS "MKL lib path = ${MKL_LIB_PATH}")
set(MKL_LIBS
${MKL_LIB_PATH}/libmkl_intel_ilp64.a
${MKL_LIB_PATH}/libmkl_gnu_thread.a
${MKL_LIB_PATH}/libmkl_core.a
)
endif ()
endmacro()
# ----------------------------------------------------------------------
# FAISS
macro(build_faiss)
message(STATUS "Building FAISS-${FAISS_VERSION} from source")
if (NOT DEFINED BUILD_FAISS_WITH_MKL)
set(BUILD_FAISS_WITH_MKL OFF)
endif ()
if (EXISTS "/proc/cpuinfo")
FILE(READ /proc/cpuinfo PROC_CPUINFO)
SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n")
STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}")
STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}")
if (NOT ${VENDOR_ID} STREQUAL "GenuineIntel")
set(BUILD_FAISS_WITH_MKL OFF)
endif ()
endif ()
set(FAISS_PREFIX "${INDEX_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep")
set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include")
set(FAISS_STATIC_LIB
@ -706,30 +711,11 @@ macro(build_faiss)
"CXXFLAGS=${EP_CXX_FLAGS}"
--without-python)
set(FAISS_CFLAGS ${EP_C_FLAGS})
set(FAISS_CXXFLAGS ${EP_CXX_FLAGS})
if (BUILD_FAISS_WITH_MKL)
find_path(MKL_LIB_PATH
NAMES "libmkl_intel_ilp64.a" "libmkl_gnu_thread.a" "libmkl_core.a"
PATH_SUFFIXES "intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64/")
if (${MKL_LIB_PATH} STREQUAL "MKL_LIB_PATH-NOTFOUND")
message(FATAL_ERROR "Could not find MKL libraries")
endif ()
message(STATUS "Build Faiss with MKL. MKL lib path = ${MKL_LIB_PATH}")
set(MKL_LIBS
${MKL_LIB_PATH}/libmkl_intel_ilp64.a
${MKL_LIB_PATH}/libmkl_gnu_thread.a
${MKL_LIB_PATH}/libmkl_core.a
)
if (FAISS_WITH_MKL)
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"CPPFLAGS=-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include"
"LDFLAGS=-L${MKL_LIB_PATH}"
)
else ()
message(STATUS "Build Faiss with OpenBlas/LAPACK")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
@ -770,7 +756,7 @@ macro(build_faiss)
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
if (NOT BUILD_FAISS_WITH_MKL)
if (NOT FAISS_WITH_MKL)
ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep)
endif ()
@ -800,7 +786,7 @@ macro(build_faiss)
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
if (NOT BUILD_FAISS_WITH_MKL)
if (NOT FAISS_WITH_MKL)
ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep)
endif ()
@ -815,7 +801,7 @@ macro(build_faiss)
IMPORTED_LOCATION "${FAISS_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}"
)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
set_target_properties(
faiss
PROPERTIES
@ -834,7 +820,9 @@ endmacro()
if (KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep)
if (NOT BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
resolve_dependency(MKL)
else ()
resolve_dependency(OpenBLAS)
get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}")

2
core/src/index/knowhere/CMakeLists.txt
View File

@ -54,7 +54,7 @@ set(depend_libs
gfortran
pthread
)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
"-Wl,--start-group \
${MKL_LIB_PATH}/libmkl_intel_ilp64.a \

2
core/src/index/thirdparty/versions.txt vendored
View File

@ -3,5 +3,5 @@ BOOST_VERSION=1.70.0
GTEST_VERSION=1.8.1
LAPACK_VERSION=v3.8.0
OPENBLAS_VERSION=v0.3.6
FAISS_VERSION=branch-0.3.0
FAISS_VERSION=1.6.0
MKL_VERSION=2019.5.281

2
core/src/index/unittest/CMakeLists.txt
View File

@ -8,7 +8,7 @@ set(depend_libs
faiss
arrow "${ARROW_LIB_DIR}/libjemalloc_pic.a"
)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
"-Wl,--start-group \
${MKL_LIB_PATH}/libmkl_intel_ilp64.a \

2
core/src/index/unittest/faiss_benchmark/CMakeLists.txt
View File

@ -15,7 +15,7 @@ if (KNOWHERE_GPU_VERSION)
faiss hdf5
arrow ${ARROW_LIB_DIR}/libjemalloc_pic.a
)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
"-Wl,--start-group \
${MKL_LIB_PATH}/libmkl_intel_ilp64.a \

2
core/src/index/unittest/faiss_ori/CMakeLists.txt
View File

@ -10,7 +10,7 @@ if (KNOWHERE_GPU_VERSION)
faiss
arrow ${ARROW_LIB_DIR}/libjemalloc_pic.a
)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
"-Wl,--start-group \
${MKL_LIB_PATH}/libmkl_intel_ilp64.a \

2
core/src/index/unittest/test_nsg/CMakeLists.txt
View File

@ -5,7 +5,7 @@
add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO)
find_package(OpenMP REQUIRED)
if (OPENMP_FOUND)
if (OpenMP_FOUND)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
else ()