admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

add find faiss

Browse Source
This commit is contained in:
Zhiru Zhu 2019-11-13 14:53:37 +08:00
parent dd7ee51222
commit bf69d10f86
11 changed files with 105 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/build.sh
View File

@ -117,7 +117,7 @@ CMAKE_CMD="cmake \
-DCUSTOMIZATION=${CUSTOMIZATION} \
-DFAISS_URL=${CUSTOMIZED_FAISS_URL} \
-DMILVUS_GPU_VERSION=${GPU_VERSION} \
-DBUILD_FAISS_WITH_MKL=${WITH_MKL} \
-DFAISS_WITH_MKL=${WITH_MKL} \
../"
echo ${CMAKE_CMD}
${CMAKE_CMD}

2
core/src/CMakeLists.txt
View File

@ -24,7 +24,7 @@ include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status)
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus)
add_subdirectory(index)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
add_compile_definitions("WITH_MKL")
endif ()

6
core/src/index/build.sh
View File

@ -5,7 +5,7 @@ BUILD_UNITTEST="OFF"
INSTALL_PREFIX=$(pwd)/cmake_build
MAKE_CLEAN="OFF"
PROFILING="OFF"
BUILD_FAISS_WITH_MKL="OFF"
FAISS_WITH_MKL="OFF"
USE_JFROG_CACHE="OFF"
while getopts "p:d:t:uhrcgmj" arg
@ -31,7 +31,7 @@ do
PROFILING="ON"
;;
m)
BUILD_FAISS_WITH_MKL="ON"
FAISS_WITH_MKL="ON"
;;
j)
USE_JFROG_CACHE="ON"
@ -74,7 +74,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
-DMILVUS_ENABLE_PROFILING=${PROFILING} \
-DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \
-DFAISS_WITH_MKL=${FAISS_WITH_MKL} \
-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
../"
echo ${CMAKE_CMD}

2
core/src/index/cmake/DefineOptionsCore.cmake
View File

@ -79,7 +79,7 @@ define_option(KNOWHERE_WITH_FAISS "Build with FAISS library" ON)
define_option(KNOWHERE_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON)
define_option(BUILD_FAISS_WITH_MKL "Build FAISS with MKL" OFF)
define_option(FAISS_WITH_MKL "Build FAISS with MKL" OFF)
#----------------------------------------------------------------------
set_option_category("Test and benchmark")

44
core/src/index/cmake/FindFAISS.cmake Normal file
View File

@ -0,0 +1,44 @@
set(FAISS_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX})
# First, find via if specified FAISS_ROOT
if (FAISS_ROOT)
find_library(FAISS_STATIC_LIB
NAMES ${FAISS_STATIC_LIB_NAME}
PATHS ${FAISS_ROOT}
PATH_SUFFIXES "lib"
NO_DEFAULT_PATH
)
find_path(FAISS_INCLUDE_DIR
NAMES "faiss/Index.h"
PATHS ${FAISS_ROOT}
NO_DEFAULT_PATH
PATH_SUFFIXES "include"
)
endif ()
find_package_handle_standard_args(FAISS REQUIRED_VARS FAISS_STATIC_LIB FAISS_INCLUDE_DIR)
if (FAISS_FOUND)
if (NOT TARGET faiss)
add_library(faiss STATIC IMPORTED)
set_target_properties(
faiss
PROPERTIES
IMPORTED_LOCATION "${FAISS_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}"
)
if (FAISS_WITH_MKL)
set_target_properties(
faiss
PROPERTIES
INTERFACE_LINK_LIBRARIES "${MKL_LIBS}")
else ()
set_target_properties(
faiss
PROPERTIES
INTERFACE_LINK_LIBRARIES "openblas;lapack")
endif ()
endif ()
endif ()

96
core/src/index/cmake/ThirdPartyPackagesCore.cmake
View File

@ -21,6 +21,7 @@ set(KNOWHERE_THIRDPARTY_DEPENDENCIES
GTest
LAPACK
OpenBLAS
MKL
)
message(STATUS "Using ${KNOWHERE_DEPENDENCY_SOURCE} approach to find dependencies")
@ -43,6 +44,8 @@ macro(build_dependency DEPENDENCY_NAME)
build_openblas()
elseif ("${DEPENDENCY_NAME}" STREQUAL "FAISS")
build_faiss()
elseif ("${DEPENDENCY_NAME}" STREQUAL "MKL")
build_mkl()
else ()
message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}")
endif ()
@ -51,9 +54,9 @@ endmacro()
macro(resolve_dependency DEPENDENCY_NAME)
if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
find_package(${DEPENDENCY_NAME} MODULE)
if(NOT ${${DEPENDENCY_NAME}_FOUND})
build_dependency(${DEPENDENCY_NAME})
endif()
if (NOT ${${DEPENDENCY_NAME}_FOUND})
build_dependency(${DEPENDENCY_NAME})
endif ()
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
build_dependency(${DEPENDENCY_NAME})
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")
@ -238,11 +241,11 @@ if (CUSTOMIZATION)
# set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
# set(FAISS_MD5 "bb30722c22390ce5f6759ccb216c1b2a") # commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0
set(FAISS_MD5 "2293cdb209c3718e3b19f3edae8b32b3") # commit-id a13c1205dc52977a9ad3b33a14efa958604a8bff branch-0.3.0
endif()
else()
endif ()
else ()
set(FAISS_SOURCE_URL "https://github.com/JinHai-CN/faiss/archive/1.6.0.tar.gz")
set(FAISS_MD5 "b02c1a53234f5acc9bea1b0c55524f50")
endif()
endif ()
message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}")
if (DEFINED ENV{KNOWHERE_ARROW_URL})
@ -673,28 +676,46 @@ if (KNOWHERE_BUILD_TESTS AND NOT TARGET googletest_ep)
include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
endif ()
# ----------------------------------------------------------------------
# MKL
macro(build_mkl)
if (FAISS_WITH_MKL)
if (EXISTS "/proc/cpuinfo")
FILE(READ /proc/cpuinfo PROC_CPUINFO)
SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n")
STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}")
STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}")
if (NOT ${VENDOR_ID} STREQUAL "GenuineIntel")
set(FAISS_WITH_MKL OFF)
endif ()
endif ()
find_path(MKL_LIB_PATH
NAMES "libmkl_intel_ilp64.a" "libmkl_gnu_thread.a" "libmkl_core.a"
PATH_SUFFIXES "intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64/")
if (${MKL_LIB_PATH} STREQUAL "MKL_LIB_PATH-NOTFOUND")
message(FATAL_ERROR "Could not find MKL libraries")
endif ()
message(STATUS "MKL lib path = ${MKL_LIB_PATH}")
set(MKL_LIBS
${MKL_LIB_PATH}/libmkl_intel_ilp64.a
${MKL_LIB_PATH}/libmkl_gnu_thread.a
${MKL_LIB_PATH}/libmkl_core.a
)
endif ()
endmacro()
# ----------------------------------------------------------------------
# FAISS
macro(build_faiss)
message(STATUS "Building FAISS-${FAISS_VERSION} from source")
if (NOT DEFINED BUILD_FAISS_WITH_MKL)
set(BUILD_FAISS_WITH_MKL OFF)
endif ()
if (EXISTS "/proc/cpuinfo")
FILE(READ /proc/cpuinfo PROC_CPUINFO)
SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n")
STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}")
STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}")
if (NOT ${VENDOR_ID} STREQUAL "GenuineIntel")
set(BUILD_FAISS_WITH_MKL OFF)
endif ()
endif ()
set(FAISS_PREFIX "${INDEX_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep")
set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include")
set(FAISS_STATIC_LIB
@ -706,30 +727,11 @@ macro(build_faiss)
"CXXFLAGS=${EP_CXX_FLAGS}"
--without-python)
set(FAISS_CFLAGS ${EP_C_FLAGS})
set(FAISS_CXXFLAGS ${EP_CXX_FLAGS})
if (BUILD_FAISS_WITH_MKL)
find_path(MKL_LIB_PATH
NAMES "libmkl_intel_ilp64.a" "libmkl_gnu_thread.a" "libmkl_core.a"
PATH_SUFFIXES "intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64/")
if (${MKL_LIB_PATH} STREQUAL "MKL_LIB_PATH-NOTFOUND")
message(FATAL_ERROR "Could not find MKL libraries")
endif ()
message(STATUS "Build Faiss with MKL. MKL lib path = ${MKL_LIB_PATH}")
set(MKL_LIBS
${MKL_LIB_PATH}/libmkl_intel_ilp64.a
${MKL_LIB_PATH}/libmkl_gnu_thread.a
${MKL_LIB_PATH}/libmkl_core.a
)
if (FAISS_WITH_MKL)
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"CPPFLAGS=-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include"
"LDFLAGS=-L${MKL_LIB_PATH}"
)
else ()
message(STATUS "Build Faiss with OpenBlas/LAPACK")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
@ -770,7 +772,7 @@ macro(build_faiss)
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
if (NOT BUILD_FAISS_WITH_MKL)
if (NOT FAISS_WITH_MKL)
ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep)
endif ()
@ -800,7 +802,7 @@ macro(build_faiss)
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
if (NOT BUILD_FAISS_WITH_MKL)
if (NOT FAISS_WITH_MKL)
ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep)
endif ()
@ -815,7 +817,7 @@ macro(build_faiss)
IMPORTED_LOCATION "${FAISS_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}"
)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
set_target_properties(
faiss
PROPERTIES
@ -834,7 +836,9 @@ endmacro()
if (KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep)
if (NOT BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
resolve_dependency(MKL)
else ()
resolve_dependency(OpenBLAS)
get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}")

2
core/src/index/knowhere/CMakeLists.txt
View File

@ -54,7 +54,7 @@ set(depend_libs
gfortran
pthread
)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
"-Wl,--start-group \
${MKL_LIB_PATH}/libmkl_intel_ilp64.a \

2
core/src/index/unittest/CMakeLists.txt
View File

@ -8,7 +8,7 @@ set(depend_libs
faiss
arrow "${ARROW_LIB_DIR}/libjemalloc_pic.a"
)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
"-Wl,--start-group \
${MKL_LIB_PATH}/libmkl_intel_ilp64.a \

2
core/src/index/unittest/faiss_benchmark/CMakeLists.txt
View File

@ -15,7 +15,7 @@ if (KNOWHERE_GPU_VERSION)
faiss hdf5
arrow ${ARROW_LIB_DIR}/libjemalloc_pic.a
)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
"-Wl,--start-group \
${MKL_LIB_PATH}/libmkl_intel_ilp64.a \

2
core/src/index/unittest/faiss_ori/CMakeLists.txt
View File

@ -10,7 +10,7 @@ if (KNOWHERE_GPU_VERSION)
faiss
arrow ${ARROW_LIB_DIR}/libjemalloc_pic.a
)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
"-Wl,--start-group \
${MKL_LIB_PATH}/libmkl_intel_ilp64.a \

2
core/src/index/unittest/test_nsg/CMakeLists.txt
View File

@ -5,7 +5,7 @@
add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO)
find_package(OpenMP REQUIRED)
if (OPENMP_FOUND)
if (OpenMP_FOUND)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
else ()