mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-01-07 19:31:51 +08:00
Merge remote-tracking branch 'upstream/branch-0.4.0' into branch-0.4.0
Former-commit-id: e46da5c74207942b8fe5d8e92c1e00b140a41ad0
This commit is contained in:
Yu Kun
commit
488c728f70
@ -26,6 +26,9 @@ Please mark all change in change log and use the ticket from JIRA.
|
||||
- MS-467 - mysql db test failed
|
||||
- MS-470 - Drop index success, which table not created
|
||||
- MS-471 - code coverage run failed
|
||||
- MS-492 - Drop index failed if index have been created with index_type: FLAT
|
||||
- MS-493 - Knowhere unittest crash
|
||||
- MS-453 - GPU search error when nprobe set more than 1024
|
||||
|
||||
## Improvement
|
||||
- MS-327 - Clean code for milvus
|
||||
@ -84,9 +87,11 @@ Please mark all change in change log and use the ticket from JIRA.
|
||||
- MS-455 - Distribute tasks by minimal cost in scheduler
|
||||
- MS-460 - Put transport speed as weight when choosing neighbour to execute task
|
||||
- MS-459 - Add cache for pick function in tasktable
|
||||
- MS-476 - Improve search performance
|
||||
- MS-482 - Change search stream transport to unary in grpc
|
||||
- MS-487 - Define metric type in CreateTable
|
||||
- MS-488 - Improve code format in scheduler
|
||||
- MS-495 - cmake: integrated knowhere
|
||||
|
||||
## New Feature
|
||||
- MS-343 - Implement ResourceMgr
|
||||
|
||||
@ -31,6 +31,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release")
|
||||
set(BUILD_TYPE "release")
|
||||
else()
|
||||
set(BUILD_TYPE "debug")
|
||||
SET(CMAKE_VERBOSE_MAKEFILE on)
|
||||
endif()
|
||||
message(STATUS "Build type = ${BUILD_TYPE}")
|
||||
|
||||
@ -70,21 +71,21 @@ set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
|
||||
find_package(CUDA)
|
||||
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda")
|
||||
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
|
||||
message("CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}")
|
||||
message("CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}")
|
||||
message(STATUS "CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}")
|
||||
message(STATUS "CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}")
|
||||
|
||||
set(CMAKE_CXX_STANDARD 14)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED on)
|
||||
|
||||
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
|
||||
message("building milvus_engine on x86 architecture")
|
||||
message(STATUS "building milvus_engine on x86 architecture")
|
||||
set(MILVUS_BUILD_ARCH x86_64)
|
||||
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
|
||||
message("building milvus_engine on ppc architecture")
|
||||
message(STATUS "building milvus_engine on ppc architecture")
|
||||
set(MILVUS_BUILD_ARCH ppc64le)
|
||||
else()
|
||||
message("unknown processor type")
|
||||
message("CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
|
||||
message(WARNING "unknown processor type")
|
||||
message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
|
||||
set(MILVUS_BUILD_ARCH unknown)
|
||||
endif()
|
||||
|
||||
@ -94,6 +95,8 @@ else()
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
|
||||
endif()
|
||||
|
||||
set(ALLOW_DUPLICATE_CUSTOM_TARGETS TRUE)
|
||||
|
||||
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
|
||||
|
||||
include(ExternalProject)
|
||||
@ -104,14 +107,6 @@ include(ThirdPartyPackages)
|
||||
include_directories(${MILVUS_SOURCE_DIR})
|
||||
link_directories(${MILVUS_BINARY_DIR})
|
||||
|
||||
if (NOT DEFINED KNOWHERE_BUILD_DIR)
|
||||
message(FATAL_ERROR "You must set environment variable KNOWHERE_BUILD_DIR")
|
||||
endif()
|
||||
message(STATUS "Build with ${KNOWHERE_BUILD_DIR}")
|
||||
include_directories(${KNOWHERE_BUILD_DIR}/include)
|
||||
include_directories(${KNOWHERE_BUILD_DIR}/include/SPTAG/AnnService)
|
||||
link_directories(${KNOWHERE_BUILD_DIR}/lib)
|
||||
|
||||
## Following should be check
|
||||
|
||||
set(MILVUS_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include)
|
||||
@ -119,7 +114,7 @@ set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
|
||||
|
||||
add_compile_definitions(PROFILER=${PROFILER})
|
||||
|
||||
message("MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}")
|
||||
message(STATUS "MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}")
|
||||
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
|
||||
ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING)
|
||||
endif()
|
||||
|
||||
@ -83,10 +83,6 @@ if [[ ! -d cmake_build ]]; then
|
||||
MAKE_CLEAN="ON"
|
||||
fi
|
||||
|
||||
pushd `pwd`/src/core
|
||||
./build.sh ${KNOWHERE_OPTIONS}
|
||||
popd
|
||||
|
||||
cd cmake_build
|
||||
|
||||
CUDA_COMPILER=/usr/local/cuda/bin/nvcc
|
||||
|
||||
@ -4,6 +4,13 @@
|
||||
# Proprietary and confidential.
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
add_subdirectory(core)
|
||||
|
||||
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
|
||||
foreach(dir ${CORE_INCLUDE_DIRS})
|
||||
include_directories(${dir})
|
||||
endforeach()
|
||||
|
||||
aux_source_directory(cache cache_files)
|
||||
aux_source_directory(config config_files)
|
||||
aux_source_directory(server server_files)
|
||||
@ -87,12 +94,6 @@ set(client_grpc_lib
|
||||
|
||||
set(third_party_libs
|
||||
knowhere
|
||||
SPTAGLibStatic
|
||||
arrow
|
||||
jemalloc_pic
|
||||
faiss
|
||||
openblas
|
||||
lapack
|
||||
easyloggingpp
|
||||
sqlite
|
||||
${client_grpc_lib}
|
||||
@ -115,15 +116,6 @@ set(third_party_libs
|
||||
cudart
|
||||
)
|
||||
|
||||
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
|
||||
set(third_party_libs ${third_party_libs}
|
||||
${MKL_LIBS})
|
||||
else()
|
||||
set(third_party_libs ${third_party_libs}
|
||||
lapack
|
||||
openblas)
|
||||
endif()
|
||||
|
||||
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
|
||||
set(third_party_libs ${third_party_libs}
|
||||
gperftools
|
||||
@ -171,13 +163,6 @@ set(server_libs
|
||||
|
||||
set(knowhere_libs
|
||||
knowhere
|
||||
SPTAGLibStatic
|
||||
arrow
|
||||
jemalloc_pic
|
||||
faiss
|
||||
openblas
|
||||
lapack
|
||||
tbb
|
||||
)
|
||||
|
||||
add_executable(milvus_server
|
||||
@ -195,8 +180,8 @@ target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_part
|
||||
install(TARGETS milvus_server DESTINATION bin)
|
||||
|
||||
install(FILES
|
||||
${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}
|
||||
${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2
|
||||
${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}
|
||||
${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2
|
||||
${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}
|
||||
${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3
|
||||
${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
cmake_minimum_required(VERSION 3.14)
|
||||
message(STATUS "---------------core--------------")
|
||||
message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
|
||||
|
||||
set(KNOWHERE_VERSION "0.1.0")
|
||||
@ -38,18 +39,16 @@ endif()
|
||||
MESSAGE(STATUS "CMAKE_CXX_FLAGS" ${CMAKE_CXX_FLAGS})
|
||||
|
||||
find_package(CUDA)
|
||||
#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
|
||||
#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
|
||||
|
||||
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
|
||||
message("building milvus_engine on x86 architecture")
|
||||
message(STATUS "building milvus_engine on x86 architecture")
|
||||
set(KNOWHERE_BUILD_ARCH x86_64)
|
||||
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
|
||||
message("building milvus_engine on ppc architecture")
|
||||
message(STATUS "building milvus_engine on ppc architecture")
|
||||
set(KNOWHERE_BUILD_ARCH ppc64le)
|
||||
else()
|
||||
message("unknown processor type")
|
||||
message("CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
|
||||
message(WARNING "unknown processor type")
|
||||
message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
|
||||
set(KNOWHERE_BUILD_ARCH unknown)
|
||||
endif()
|
||||
|
||||
@ -60,14 +59,21 @@ else()
|
||||
endif()
|
||||
message(STATUS "Build type = ${BUILD_TYPE}")
|
||||
|
||||
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
|
||||
set(CORE_SOURCE_DIR ${PROJECT_SOURCE_DIR})
|
||||
set(CORE_BINARY_DIR ${PROJECT_BINARY_DIR})
|
||||
message(STATUS "Core source dir: ${PROJECT_SOURCE_DIR}")
|
||||
message(STATUS "Core binary dir: ${PROJECT_BINARY_DIR}")
|
||||
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CORE_SOURCE_DIR}/cmake")
|
||||
|
||||
include(ExternalProject)
|
||||
include(DefineOptions)
|
||||
include(BuildUtils)
|
||||
include(ThirdPartyPackages)
|
||||
include(DefineOptionsCore)
|
||||
include(BuildUtilsCore)
|
||||
include(ThirdPartyPackagesCore)
|
||||
|
||||
add_subdirectory(src)
|
||||
|
||||
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
|
||||
|
||||
if(BUILD_UNIT_TEST STREQUAL "ON")
|
||||
add_subdirectory(test)
|
||||
endif()
|
||||
|
||||
@ -75,7 +75,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path)
|
||||
"Extracting ${package_file} to ${install_path}"
|
||||
COMMAND
|
||||
${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path}
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
|
||||
WORKING_DIRECTORY ${CORE_BINARY_DIR}
|
||||
)
|
||||
|
||||
ExternalProject_Add_StepTargets(${project_name} extract)
|
||||
@ -104,7 +104,7 @@ macro(config_summary)
|
||||
message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}")
|
||||
if(${CMAKE_EXPORT_COMPILE_COMMANDS})
|
||||
message(
|
||||
STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json")
|
||||
STATUS " Compile commands: ${CORE_BINARY_DIR}/compile_commands.json")
|
||||
endif()
|
||||
|
||||
foreach(category ${KNOWHERE_OPTION_CATEGORIES})
|
||||
@ -6,7 +6,6 @@
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
@ -18,7 +17,6 @@
|
||||
set(KNOWHERE_THIRDPARTY_DEPENDENCIES
|
||||
|
||||
ARROW
|
||||
BOOST
|
||||
FAISS
|
||||
GTest
|
||||
LAPACK
|
||||
@ -53,11 +51,8 @@ endmacro()
|
||||
macro(resolve_dependency DEPENDENCY_NAME)
|
||||
if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
|
||||
#message(STATUS "Finding ${DEPENDENCY_NAME} package")
|
||||
# find_package(${DEPENDENCY_NAME} QUIET)
|
||||
# if (NOT ${DEPENDENCY_NAME}_FOUND)
|
||||
#message(STATUS "${DEPENDENCY_NAME} package not found")
|
||||
build_dependency(${DEPENDENCY_NAME})
|
||||
# endif ()
|
||||
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
|
||||
build_dependency(${DEPENDENCY_NAME})
|
||||
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")
|
||||
@ -120,7 +115,7 @@ endif (UNIX)
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# thirdparty directory
|
||||
set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
|
||||
set(THIRDPARTY_DIR "${CORE_SOURCE_DIR}/thirdparty")
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# JFrog
|
||||
@ -202,14 +197,6 @@ endif()
|
||||
|
||||
set(MAKE_BUILD_ARGS "-j8")
|
||||
|
||||
## Using make -j in sub-make is fragile
|
||||
## see discussion https://github.com/apache/KNOWHERE/pull/2779
|
||||
#if(${CMAKE_GENERATOR} MATCHES "Makefiles")
|
||||
# set(MAKE_BUILD_ARGS "")
|
||||
#else()
|
||||
# # limit the maximum number of jobs for ninja
|
||||
# set(MAKE_BUILD_ARGS "-j4")
|
||||
#endif()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Find pthreads
|
||||
@ -243,22 +230,11 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
|
||||
set(${_LIB_NAME} "${_LIB_VERSION}")
|
||||
endforeach()
|
||||
|
||||
if(DEFINED ENV{KNOWHERE_BOOST_URL})
|
||||
set(BOOST_SOURCE_URL "$ENV{KNOWHERE_BOOST_URL}")
|
||||
else()
|
||||
string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION})
|
||||
set(BOOST_SOURCE_URL
|
||||
"https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz"
|
||||
)
|
||||
endif()
|
||||
|
||||
if(DEFINED ENV{KNOWHERE_FAISS_URL})
|
||||
set(FAISS_SOURCE_URL "$ENV{KNOWHERE_FAISS_URL}")
|
||||
else()
|
||||
set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz")
|
||||
# set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz")
|
||||
# set(FAISS_SOURCE_URL "${CMAKE_SOURCE_DIR}/thirdparty/faiss-1.5.3")
|
||||
message(STATUS ${FAISS_SOURCE_URL})
|
||||
message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}")
|
||||
endif()
|
||||
# set(FAISS_MD5 "a589663865a8558205533c8ac414278c")
|
||||
# set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c
|
||||
@ -297,28 +273,18 @@ set(OPENBLAS_MD5 "8a110a25b819a4b94e8a9580702b6495")
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# ARROW
|
||||
set(ARROW_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp")
|
||||
|
||||
macro(build_arrow)
|
||||
message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source")
|
||||
set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp")
|
||||
set(ARROW_STATIC_LIB_NAME arrow)
|
||||
# set(PARQUET_STATIC_LIB_NAME parquet)
|
||||
# set(ARROW_CUDA_STATIC_LIB_NAME arrow_cuda)
|
||||
set(ARROW_STATIC_LIB
|
||||
set(ARROW_STATIC_LIB
|
||||
"${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
|
||||
)
|
||||
# set(PARQUET_STATIC_LIB
|
||||
# "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
|
||||
# )
|
||||
# set(ARROW_CUDA_STATIC_LIB
|
||||
# "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_CUDA_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
|
||||
# )
|
||||
set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")
|
||||
set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")
|
||||
|
||||
set(ARROW_CMAKE_ARGS
|
||||
${EP_COMMON_CMAKE_ARGS}
|
||||
# "-DARROW_THRIFT_URL=${THRIFT_SOURCE_URL}"
|
||||
#"env ARROW_THRIFT_URL=${THRIFT_SOURCE_URL}"
|
||||
-DARROW_BUILD_STATIC=ON
|
||||
-DARROW_BUILD_SHARED=OFF
|
||||
-DARROW_PARQUET=OFF
|
||||
@ -328,8 +294,7 @@ macro(build_arrow)
|
||||
-DCMAKE_BUILD_TYPE=Release
|
||||
-DARROW_DEPENDENCY_SOURCE=BUNDLED) #Build all arrow dependencies from source instead of calling find_package first
|
||||
|
||||
# set($ENV{ARROW_THRIFT_URL} ${THRIFT_SOURCE_URL})
|
||||
|
||||
|
||||
if(USE_JFROG_CACHE STREQUAL "ON")
|
||||
execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID)
|
||||
if(${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+")
|
||||
@ -362,13 +327,13 @@ macro(build_arrow)
|
||||
"${ARROW_STATIC_LIB}"
|
||||
)
|
||||
|
||||
ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL})
|
||||
ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL})
|
||||
else()
|
||||
file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status)
|
||||
list(GET status 0 status_code)
|
||||
message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
|
||||
if (status_code EQUAL 0)
|
||||
ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
|
||||
ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR})
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
@ -402,10 +367,9 @@ macro(build_arrow)
|
||||
set_target_properties(arrow
|
||||
PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}")
|
||||
# INTERFACE_LINK_LIBRARIES thrift)
|
||||
add_dependencies(arrow arrow_ep)
|
||||
add_dependencies(arrow arrow_ep)
|
||||
|
||||
set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep")
|
||||
set(JEMALLOC_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep")
|
||||
|
||||
add_custom_command(TARGET arrow_ep POST_BUILD
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_PREFIX}/lib/
|
||||
@ -414,7 +378,7 @@ macro(build_arrow)
|
||||
|
||||
endmacro()
|
||||
|
||||
if(KNOWHERE_WITH_ARROW)
|
||||
if(KNOWHERE_WITH_ARROW AND NOT TARGET arrow_ep)
|
||||
|
||||
resolve_dependency(ARROW)
|
||||
|
||||
@ -422,191 +386,12 @@ if(KNOWHERE_WITH_ARROW)
|
||||
include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
|
||||
endif()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Add Boost dependencies (code adapted from Apache Kudu (incubating))
|
||||
|
||||
set(Boost_USE_MULTITHREADED ON)
|
||||
if(MSVC AND KNOWHERE_USE_STATIC_CRT)
|
||||
set(Boost_USE_STATIC_RUNTIME ON)
|
||||
endif()
|
||||
set(Boost_ADDITIONAL_VERSIONS
|
||||
"1.70.0"
|
||||
"1.70"
|
||||
"1.69.0"
|
||||
"1.69"
|
||||
"1.68.0"
|
||||
"1.68"
|
||||
"1.67.0"
|
||||
"1.67"
|
||||
"1.66.0"
|
||||
"1.66"
|
||||
"1.65.0"
|
||||
"1.65"
|
||||
"1.64.0"
|
||||
"1.64"
|
||||
"1.63.0"
|
||||
"1.63"
|
||||
"1.62.0"
|
||||
"1.61"
|
||||
"1.61.0"
|
||||
"1.62"
|
||||
"1.60.0"
|
||||
"1.60")
|
||||
|
||||
# TODO
|
||||
if(KNOWHERE_BOOST_VENDORED)
|
||||
# system thread serialization wserialization regex
|
||||
set(BOOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/boost_ep-prefix/src/boost_ep")
|
||||
set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib")
|
||||
set(BOOST_BUILD_LINK "static")
|
||||
set(BOOST_STATIC_SYSTEM_LIBRARY
|
||||
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}"
|
||||
)
|
||||
set(BOOST_STATIC_FILESYSTEM_LIBRARY
|
||||
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}"
|
||||
)
|
||||
set(BOOST_STATIC_SERIALIZATION_LIBRARY
|
||||
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_serialization${CMAKE_STATIC_LIBRARY_SUFFIX}"
|
||||
)
|
||||
set(BOOST_STATIC_WSERIALIZATION_LIBRARY
|
||||
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_wserialization${CMAKE_STATIC_LIBRARY_SUFFIX}"
|
||||
)
|
||||
set(BOOST_STATIC_REGEX_LIBRARY
|
||||
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}"
|
||||
)
|
||||
set(BOOST_STATIC_THREAD_LIBRARY
|
||||
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_thread${CMAKE_STATIC_LIBRARY_SUFFIX}"
|
||||
)
|
||||
set(BOOST_SYSTEM_LIBRARY boost_system_static)
|
||||
set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
|
||||
set(BOOST_SERIALIZATION_LIBRARY boost_serialization_static)
|
||||
set(BOOST_WSERIALIZATION_LIBRARY boost_wserialization_static)
|
||||
set(BOOST_REGEX_LIBRARY boost_regex_static)
|
||||
set(BOOST_THREAD_LIBRARY boost_thread_static)
|
||||
|
||||
if(KNOWHERE_BOOST_HEADER_ONLY)
|
||||
set(BOOST_BUILD_PRODUCTS)
|
||||
set(BOOST_CONFIGURE_COMMAND "")
|
||||
set(BOOST_BUILD_COMMAND "")
|
||||
else()
|
||||
set(BOOST_BUILD_PRODUCTS ${BOOST_STATIC_SYSTEM_LIBRARY}
|
||||
${BOOST_STATIC_FILESYSTEM_LIBRARY} ${BOOST_STATIC_SERIALIZATION_LIBRARY}
|
||||
${BOOST_STATIC_WSERIALIZATION_LIBRARY} ${BOOST_STATIC_REGEX_LIBRARY}
|
||||
${BOOST_STATIC_THREAD_LIBRARY})
|
||||
set(BOOST_CONFIGURE_COMMAND "./bootstrap.sh" "--prefix=${BOOST_PREFIX}"
|
||||
"--with-libraries=filesystem,serialization,wserialization,system,thread,regex")
|
||||
if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
|
||||
set(BOOST_BUILD_VARIANT "debug")
|
||||
else()
|
||||
set(BOOST_BUILD_VARIANT "release")
|
||||
endif()
|
||||
set(BOOST_BUILD_COMMAND
|
||||
"./b2"
|
||||
"link=${BOOST_BUILD_LINK}"
|
||||
"variant=${BOOST_BUILD_VARIANT}"
|
||||
"cxxflags=-fPIC")
|
||||
|
||||
add_thirdparty_lib(boost_system STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}")
|
||||
|
||||
add_thirdparty_lib(boost_filesystem STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}")
|
||||
|
||||
add_thirdparty_lib(boost_serialization STATIC_LIB "${BOOST_STATIC_SERIALIZATION_LIBRARY}")
|
||||
|
||||
add_thirdparty_lib(boost_wserialization STATIC_LIB "${BOOST_STATIC_WSERIALIZATION_LIBRARY}")
|
||||
|
||||
add_thirdparty_lib(boost_regex STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}")
|
||||
|
||||
add_thirdparty_lib(boost_thread STATIC_LIB "${BOOST_STATIC_THREAD_LIBRARY}")
|
||||
|
||||
set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_SERIALIZATION_LIBRARY}
|
||||
${BOOST_WSERIALIZATION_LIBRARY} ${BOOST_REGEX_LIBRARY} ${BOOST_THREAD_LIBRARY})
|
||||
endif()
|
||||
externalproject_add(boost_ep
|
||||
URL
|
||||
${BOOST_SOURCE_URL}
|
||||
BUILD_BYPRODUCTS
|
||||
${BOOST_BUILD_PRODUCTS}
|
||||
BUILD_IN_SOURCE
|
||||
1
|
||||
CONFIGURE_COMMAND
|
||||
${BOOST_CONFIGURE_COMMAND}
|
||||
BUILD_COMMAND
|
||||
${BOOST_BUILD_COMMAND}
|
||||
INSTALL_COMMAND
|
||||
""
|
||||
${EP_LOG_OPTIONS})
|
||||
set(Boost_INCLUDE_DIR "${BOOST_PREFIX}")
|
||||
set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}")
|
||||
add_dependencies(boost_system_static boost_ep)
|
||||
add_dependencies(boost_filesystem_static boost_ep)
|
||||
add_dependencies(boost_serialization_static boost_ep)
|
||||
add_dependencies(boost_wserialization_static boost_ep)
|
||||
add_dependencies(boost_regex_static boost_ep)
|
||||
add_dependencies(boost_thread_static boost_ep)
|
||||
|
||||
#else()
|
||||
# if(MSVC)
|
||||
# # disable autolinking in boost
|
||||
# add_definitions(-DBOOST_ALL_NO_LIB)
|
||||
# endif()
|
||||
|
||||
# if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT)
|
||||
# # In older versions of CMake (such as 3.2), the system paths for Boost will
|
||||
# # In older versions of CMake (such as 3.2), the system paths for Boost will
|
||||
# # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT
|
||||
# set(Boost_NO_SYSTEM_PATHS ON)
|
||||
# endif()
|
||||
|
||||
# if(KNOWHERE_BOOST_USE_SHARED)
|
||||
# # Find shared Boost libraries.
|
||||
# set(Boost_USE_STATIC_LIBS OFF)
|
||||
# set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS})
|
||||
# set(BUILD_SHARED_LIBS ON)
|
||||
#
|
||||
# if(MSVC)
|
||||
# # force all boost libraries to dynamic link
|
||||
# add_definitions(-DBOOST_ALL_DYN_LINK)
|
||||
# endif()
|
||||
#
|
||||
# if(KNOWHERE_BOOST_HEADER_ONLY)
|
||||
# find_package(Boost REQUIRED)
|
||||
# else()
|
||||
# find_package(Boost COMPONENTS serialization system filesystem REQUIRED)
|
||||
# set(BOOST_SYSTEM_LIBRARY Boost::system)
|
||||
# set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem)
|
||||
# set(BOOST_SERIALIZATION_LIBRARY Boost::serialization)
|
||||
# set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY})
|
||||
# endif()
|
||||
# set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP})
|
||||
# unset(BUILD_SHARED_LIBS_KEEP)
|
||||
# else()
|
||||
# # Find static boost headers and libs
|
||||
# # TODO Differentiate here between release and debug builds
|
||||
# set(Boost_USE_STATIC_LIBS ON)
|
||||
# if(KNOWHERE_BOOST_HEADER_ONLY)
|
||||
# find_package(Boost REQUIRED)
|
||||
# else()
|
||||
# find_package(Boost COMPONENTS serialization system filesystem REQUIRED)
|
||||
# set(BOOST_SYSTEM_LIBRARY Boost::system)
|
||||
# set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem)
|
||||
# set(BOOST_SERIALIZATION_LIBRARY Boost::serialization)
|
||||
# set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY})
|
||||
# endif()
|
||||
# endif()
|
||||
endif()
|
||||
|
||||
#message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR})
|
||||
#message(STATUS "Boost libraries: " ${Boost_LIBRARIES})
|
||||
|
||||
include_directories(SYSTEM ${Boost_INCLUDE_DIR})
|
||||
link_directories(SYSTEM ${BOOST_LIB_DIR})
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# OpenBLAS
|
||||
|
||||
macro(build_openblas)
|
||||
message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source")
|
||||
set(OPENBLAS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep")
|
||||
set(OPENBLAS_PREFIX "${CORE_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep")
|
||||
set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include")
|
||||
set(OPENBLAS_STATIC_LIB
|
||||
"${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}")
|
||||
@ -639,13 +424,13 @@ macro(build_openblas)
|
||||
BUILD_BYPRODUCTS
|
||||
${OPENBLAS_STATIC_LIB})
|
||||
|
||||
ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL})
|
||||
ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL})
|
||||
else()
|
||||
file(DOWNLOAD ${OPENBLAS_CACHE_URL} ${OPENBLAS_CACHE_PACKAGE_PATH} STATUS status)
|
||||
list(GET status 0 status_code)
|
||||
message(STATUS "DOWNLOADING FROM ${OPENBLAS_CACHE_URL} TO ${OPENBLAS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
|
||||
if (status_code EQUAL 0)
|
||||
ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
|
||||
ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR})
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
@ -678,20 +463,12 @@ macro(build_openblas)
|
||||
add_dependencies(openblas openblas_ep)
|
||||
endmacro()
|
||||
|
||||
#if(KNOWHERE_WITH_OPENBLAS)
|
||||
# resolve_dependency(OpenBLAS)
|
||||
#
|
||||
# get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
|
||||
# include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}")
|
||||
# link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib)
|
||||
#endif()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# LAPACK
|
||||
|
||||
macro(build_lapack)
|
||||
message(STATUS "Building LAPACK-${LAPACK_VERSION} from source")
|
||||
set(LAPACK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep")
|
||||
set(LAPACK_PREFIX "${CORE_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep")
|
||||
set(LAPACK_INCLUDE_DIR "${LAPACK_PREFIX}/include")
|
||||
set(LAPACK_STATIC_LIB
|
||||
"${LAPACK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lapack${CMAKE_STATIC_LIBRARY_SUFFIX}")
|
||||
@ -723,13 +500,13 @@ macro(build_lapack)
|
||||
BUILD_BYPRODUCTS
|
||||
${LAPACK_STATIC_LIB})
|
||||
|
||||
ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL})
|
||||
ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL})
|
||||
else()
|
||||
file(DOWNLOAD ${LAPACK_CACHE_URL} ${LAPACK_CACHE_PACKAGE_PATH} STATUS status)
|
||||
list(GET status 0 status_code)
|
||||
message(STATUS "DOWNLOADING FROM ${LAPACK_CACHE_URL} TO ${LAPACK_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
|
||||
if (status_code EQUAL 0)
|
||||
ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
|
||||
ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR})
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
@ -756,14 +533,6 @@ macro(build_lapack)
|
||||
add_dependencies(lapack lapack_ep)
|
||||
endmacro()
|
||||
|
||||
#if(KNOWHERE_WITH_LAPACK)
|
||||
# resolve_dependency(LAPACK)
|
||||
#
|
||||
# get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES)
|
||||
# include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}")
|
||||
# link_directories(SYSTEM ${LAPACK_PREFIX}/lib)
|
||||
#endif()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Google gtest
|
||||
|
||||
@ -780,7 +549,7 @@ macro(build_gtest)
|
||||
-Wno-ignored-attributes)
|
||||
endif()
|
||||
|
||||
set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
|
||||
set(GTEST_PREFIX "${CORE_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
|
||||
set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
|
||||
set(GTEST_STATIC_LIB
|
||||
"${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
|
||||
@ -822,13 +591,13 @@ macro(build_gtest)
|
||||
${GTEST_CMAKE_ARGS}
|
||||
${EP_LOG_OPTIONS})
|
||||
|
||||
ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL})
|
||||
ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL})
|
||||
else()
|
||||
file(DOWNLOAD ${GTEST_CACHE_URL} ${GTEST_CACHE_PACKAGE_PATH} STATUS status)
|
||||
list(GET status 0 status_code)
|
||||
message(STATUS "DOWNLOADING FROM ${GTEST_CACHE_URL} TO ${GTEST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
|
||||
if (status_code EQUAL 0)
|
||||
ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
|
||||
ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR})
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
@ -871,8 +640,7 @@ macro(build_gtest)
|
||||
|
||||
endmacro()
|
||||
|
||||
if (KNOWHERE_BUILD_TESTS)
|
||||
#message(STATUS "Resolving gtest dependency")
|
||||
if (KNOWHERE_BUILD_TESTS AND NOT TARGET googletest_ep)
|
||||
resolve_dependency(GTest)
|
||||
|
||||
if(NOT GTEST_VENDORED)
|
||||
@ -889,17 +657,12 @@ endif()
|
||||
|
||||
macro(build_faiss)
|
||||
message(STATUS "Building FAISS-${FAISS_VERSION} from source")
|
||||
set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep")
|
||||
set(FAISS_PREFIX "${CORE_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep")
|
||||
set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include")
|
||||
set(FAISS_STATIC_LIB
|
||||
"${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}")
|
||||
|
||||
# add_custom_target(faiss_dependencies)
|
||||
# add_dependencies(faiss_dependencies openblas_ep)
|
||||
# add_dependencies(faiss_dependencies openblas)
|
||||
# get_target_property(FAISS_OPENBLAS_LIB_DIR openblas IMPORTED_LOCATION)
|
||||
# get_filename_component(FAISS_OPENBLAS_LIB "${FAISS_OPENBLAS_LIB_DIR}" DIRECTORY)
|
||||
|
||||
|
||||
set(FAISS_CONFIGURE_ARGS
|
||||
"--prefix=${FAISS_PREFIX}"
|
||||
"CFLAGS=${EP_C_FLAGS}"
|
||||
@ -907,13 +670,7 @@ macro(build_faiss)
|
||||
"LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib -lopenblas -llapack"
|
||||
--without-python)
|
||||
|
||||
# if(OPENBLAS_STATIC_LIB)
|
||||
# set(OPENBLAS_LIBRARY ${OPENBLAS_STATIC_LIB})
|
||||
# else()
|
||||
# set(OPENBLAS_LIBRARY ${OPENBLAS_SHARED_LIB})
|
||||
# endif()
|
||||
# set(FAISS_DEPENDENCIES ${FAISS_DEPENDENCIES} ${OPENBLAS_LIBRARY})
|
||||
|
||||
|
||||
if(${KNOWHERE_WITH_FAISS_GPU_VERSION} STREQUAL "ON")
|
||||
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
|
||||
"--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
|
||||
@ -924,9 +681,7 @@ macro(build_faiss)
|
||||
endif()
|
||||
|
||||
if(USE_JFROG_CACHE STREQUAL "ON")
|
||||
# Check_Last_Modify("${CMAKE_SOURCE_DIR}/thirdparty/faiss_cache_check_lists.txt" "${CMAKE_SOURCE_DIR}" FAISS_LAST_MODIFIED_COMMIT_ID)
|
||||
string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}")
|
||||
# string(MD5 FAISS_COMBINE_MD5 "${FAISS_LAST_MODIFIED_COMMIT_ID}${LAPACK_MD5}${OPENBLAS_MD5}")
|
||||
set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMBINE_MD5}.tar.gz")
|
||||
set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}")
|
||||
set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}")
|
||||
@ -952,13 +707,13 @@ macro(build_faiss)
|
||||
|
||||
ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep)
|
||||
|
||||
ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL})
|
||||
ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL})
|
||||
else()
|
||||
file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status)
|
||||
list(GET status 0 status_code)
|
||||
message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
|
||||
if (status_code EQUAL 0)
|
||||
ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
|
||||
ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR})
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
@ -990,14 +745,10 @@ macro(build_faiss)
|
||||
INTERFACE_LINK_LIBRARIES "openblas;lapack" )
|
||||
|
||||
add_dependencies(faiss faiss_ep)
|
||||
#add_dependencies(faiss openblas_ep)
|
||||
#add_dependencies(faiss lapack_ep)
|
||||
#target_link_libraries(faiss ${OPENBLAS_PREFIX}/lib)
|
||||
#target_link_libraries(faiss ${LAPACK_PREFIX}/lib)
|
||||
|
||||
endmacro()
|
||||
|
||||
if(KNOWHERE_WITH_FAISS)
|
||||
if(KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep)
|
||||
|
||||
resolve_dependency(OpenBLAS)
|
||||
get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
|
||||
@ -1012,8 +763,5 @@ if(KNOWHERE_WITH_FAISS)
|
||||
resolve_dependency(FAISS)
|
||||
get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES)
|
||||
include_directories(SYSTEM "${FAISS_INCLUDE_DIR}")
|
||||
# include_directories(SYSTEM "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/")
|
||||
# link_directories(SYSTEM ${FAISS_PREFIX}/)
|
||||
link_directories(SYSTEM ${FAISS_PREFIX}/lib/)
|
||||
# link_directories(SYSTEM ${FAISS_PREFIX}/gpu/)
|
||||
endif()
|
||||
@ -9,7 +9,7 @@ namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
struct Resource {
|
||||
Resource(std::shared_ptr<faiss::gpu::StandardGpuResources> &r): faiss_res(r) {
|
||||
explicit Resource(std::shared_ptr<faiss::gpu::StandardGpuResources> &r): faiss_res(r) {
|
||||
static int64_t global_id = 0;
|
||||
id = global_id++;
|
||||
}
|
||||
@ -32,6 +32,11 @@ class FaissGpuResourceMgr {
|
||||
static FaissGpuResourceMgr &
|
||||
GetInstance();
|
||||
|
||||
// Free gpu resource, avoid cudaGetDevice error when deallocate.
|
||||
// this func should be invoke before main return
|
||||
void
|
||||
Free();
|
||||
|
||||
void
|
||||
AllocateTempMem(ResPtr &resource, const int64_t& device_id, const int64_t& size);
|
||||
|
||||
|
||||
@ -1,16 +1,16 @@
|
||||
set(TBB_DIR ${CMAKE_SOURCE_DIR}/thirdparty/tbb)
|
||||
set(TBB_DIR ${CORE_SOURCE_DIR}/thirdparty/tbb)
|
||||
set(TBB_LIBRARIES ${TBB_DIR}/libtbb.so)
|
||||
include_directories(${TBB_DIR}/include)
|
||||
|
||||
include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
|
||||
link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64)
|
||||
|
||||
include_directories(${CMAKE_SOURCE_DIR}/include)
|
||||
include_directories(${CMAKE_SOURCE_DIR}/thirdparty)
|
||||
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
|
||||
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
|
||||
include_directories(${CORE_SOURCE_DIR}/include)
|
||||
include_directories(${CORE_SOURCE_DIR}/thirdparty)
|
||||
include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
|
||||
include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
|
||||
|
||||
set(SPTAG_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/SPTAG)
|
||||
set(SPTAG_SOURCE_DIR ${CORE_SOURCE_DIR}/thirdparty/SPTAG)
|
||||
file(GLOB HDR_FILES
|
||||
${SPTAG_SOURCE_DIR}/AnnService/inc/Core/*.h
|
||||
${SPTAG_SOURCE_DIR}/AnnService/inc/Core/Common/*.h
|
||||
@ -24,9 +24,9 @@ file(GLOB SRC_FILES
|
||||
${SPTAG_SOURCE_DIR}/AnnService/src/Core/KDT/*.cpp
|
||||
${SPTAG_SOURCE_DIR}/AnnService/src/Helper/*.cpp)
|
||||
|
||||
#add_library(SPTAGLib SHARED ${SRC_FILES} ${HDR_FILES})
|
||||
#target_link_libraries(SPTAGLib ${TBB_LIBRARIES})
|
||||
add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES})
|
||||
if(NOT TARGET SPTAGLibStatic)
|
||||
add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES})
|
||||
endif()
|
||||
|
||||
set(external_srcs
|
||||
knowhere/adapter/sptag.cpp
|
||||
@ -51,14 +51,13 @@ set(index_srcs
|
||||
)
|
||||
|
||||
set(depend_libs
|
||||
# libtcmalloc.a
|
||||
SPTAGLibStatic
|
||||
${TBB_LIBRARIES}
|
||||
faiss
|
||||
openblas
|
||||
lapack
|
||||
arrow
|
||||
jemalloc_pic
|
||||
${ARROW_PREFIX}/lib/libjemalloc_pic.a
|
||||
cudart
|
||||
cublas
|
||||
gomp
|
||||
@ -66,30 +65,19 @@ set(depend_libs
|
||||
pthread
|
||||
)
|
||||
|
||||
add_library(
|
||||
knowhere STATIC
|
||||
${external_srcs}
|
||||
${index_srcs}
|
||||
)
|
||||
#target_compile_options(knowhere PUBLIC "-fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free")
|
||||
if(NOT TARGET knowhere)
|
||||
add_library(
|
||||
knowhere STATIC
|
||||
${external_srcs}
|
||||
${index_srcs}
|
||||
)
|
||||
endif()
|
||||
|
||||
target_link_libraries(
|
||||
knowhere
|
||||
${depend_libs}
|
||||
)
|
||||
|
||||
#add_library(
|
||||
# knowhereS SHARED
|
||||
# ${external_srcs}
|
||||
# ${index_srcs}
|
||||
#)
|
||||
#
|
||||
#target_link_libraries(
|
||||
# knowhereS
|
||||
## ${TBB_LIBRARIES}
|
||||
# ${depend_libs}
|
||||
#)
|
||||
|
||||
INSTALL(TARGETS
|
||||
knowhere
|
||||
SPTAGLibStatic
|
||||
@ -98,7 +86,6 @@ INSTALL(TARGETS
|
||||
|
||||
INSTALL(FILES
|
||||
${ARROW_STATIC_LIB}
|
||||
# ${PARQUET_STATIC_LIB}
|
||||
${ARROW_PREFIX}/lib/libjemalloc_pic.a
|
||||
${FAISS_STATIC_LIB}
|
||||
${LAPACK_STATIC_LIB}
|
||||
@ -112,24 +99,35 @@ INSTALL(FILES ${OPENBLAS_REAL_STATIC_LIB}
|
||||
DESTINATION lib
|
||||
)
|
||||
|
||||
INSTALL(FILES ${CMAKE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2
|
||||
# RENAME "libtbb.so.2"
|
||||
INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2
|
||||
DESTINATION lib
|
||||
)
|
||||
INSTALL(FILES ${CMAKE_SOURCE_DIR}/thirdparty/tbb/libtbb.so
|
||||
# RENAME "libtbb.so"
|
||||
INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so
|
||||
DESTINATION lib
|
||||
)
|
||||
|
||||
set(CORE_INCLUDE_DIRS
|
||||
${CORE_SOURCE_DIR}/include
|
||||
${CORE_SOURCE_DIR}/thirdparty
|
||||
${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService
|
||||
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include
|
||||
${ARROW_INCLUDE_DIR}
|
||||
${FAISS_INCLUDE_DIR}
|
||||
${OPENBLAS_INCLUDE_DIR}
|
||||
${LAPACK_INCLUDE_DIR}
|
||||
${CORE_SOURCE_DIR}/thirdparty/tbb/include
|
||||
)
|
||||
|
||||
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
|
||||
|
||||
INSTALL(DIRECTORY
|
||||
${CMAKE_SOURCE_DIR}/include/knowhere
|
||||
${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons
|
||||
${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext
|
||||
${CORE_SOURCE_DIR}/include/knowhere
|
||||
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons
|
||||
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext
|
||||
${ARROW_INCLUDE_DIR}/arrow
|
||||
# ${ARROW_INCLUDE_DIR}/parquet
|
||||
${FAISS_PREFIX}/include/faiss
|
||||
${OPENBLAS_INCLUDE_DIR}/
|
||||
${CMAKE_SOURCE_DIR}/thirdparty/tbb/include/tbb
|
||||
${CORE_SOURCE_DIR}/thirdparty/tbb/include/tbb
|
||||
DESTINATION
|
||||
include)
|
||||
|
||||
|
||||
@ -282,7 +282,7 @@ void FaissGpuResourceMgr::InitResource() {
|
||||
for(auto& device : devices_params_) {
|
||||
auto& resource_vec = idle_[device.first];
|
||||
|
||||
for (int i = 0; i < device.second.resource_num; ++i) {
|
||||
for (int64_t i = 0; i < device.second.resource_num; ++i) {
|
||||
auto res = std::make_shared<faiss::gpu::StandardGpuResources>();
|
||||
|
||||
// TODO(linxj): enable set pinned memory
|
||||
@ -351,6 +351,18 @@ void FaissGpuResourceMgr::MoveToIdle(const int64_t &device_id, const ResPtr &res
|
||||
idle_[device_id].insert(it, res);
|
||||
}
|
||||
|
||||
void FaissGpuResourceMgr::Free() {
|
||||
for (auto &item : in_use_) {
|
||||
auto& res_vec = item.second;
|
||||
res_vec.clear();
|
||||
}
|
||||
for (auto &item : idle_) {
|
||||
auto& res_vec = item.second;
|
||||
res_vec.clear();
|
||||
}
|
||||
is_init = false;
|
||||
}
|
||||
|
||||
void GPUIndex::SetGpuDevice(const int &gpu_id) {
|
||||
gpu_id_ = gpu_id;
|
||||
}
|
||||
|
||||
@ -1,17 +1,19 @@
|
||||
include_directories(${CMAKE_SOURCE_DIR}/thirdparty)
|
||||
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
|
||||
include_directories(${CMAKE_SOURCE_DIR}/include)
|
||||
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
|
||||
include_directories(${CORE_SOURCE_DIR}/thirdparty)
|
||||
include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
|
||||
include_directories(${CORE_SOURCE_DIR}/include)
|
||||
include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
|
||||
include_directories(/usr/local/cuda/include)
|
||||
link_directories(/usr/local/cuda/lib64)
|
||||
link_directories(${CMAKE_SOURCE_DIR}/thirdparty/tbb)
|
||||
link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb)
|
||||
|
||||
set(unittest_libs
|
||||
gtest gmock gtest_main gmock_main)
|
||||
|
||||
message(STATUS "arrow prefix: ${ARROW_PREFIX}")
|
||||
message(STATUS "libjemalloc_pic path: ${ARROW_PREFIX}/lib/libjemalloc_pic.a")
|
||||
set(depend_libs
|
||||
faiss openblas lapack
|
||||
arrow jemalloc_pic
|
||||
arrow "${ARROW_PREFIX}/lib/libjemalloc_pic.a"
|
||||
tbb
|
||||
)
|
||||
|
||||
@ -22,57 +24,55 @@ set(basic_libs
|
||||
|
||||
|
||||
#<IVF-TEST>
|
||||
#<<<<<<<<<<
|
||||
set(ivf_srcs
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
|
||||
utils.cpp
|
||||
)
|
||||
add_executable(test_ivf test_ivf.cpp ${ivf_srcs})
|
||||
if(NOT TARGET test_ivf)
|
||||
add_executable(test_ivf test_ivf.cpp ${ivf_srcs})
|
||||
endif()
|
||||
target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
#<<<<<<<<<<
|
||||
|
||||
#<IDMAP-TEST>
|
||||
#<<<<<<<<<<
|
||||
set(idmap_srcs
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
|
||||
utils.cpp
|
||||
)
|
||||
add_executable(test_idmap test_idmap.cpp ${idmap_srcs})
|
||||
if(NOT TARGET test_idmap)
|
||||
add_executable(test_idmap test_idmap.cpp ${idmap_srcs})
|
||||
endif()
|
||||
target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
#<<<<<<<<<<
|
||||
|
||||
|
||||
#<KDT-TEST>
|
||||
#<<<<<<<<<<
|
||||
set(kdt_srcs
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
|
||||
utils.cpp
|
||||
)
|
||||
add_executable(test_kdt test_kdt.cpp ${kdt_srcs})
|
||||
if(NOT TARGET test_kdt)
|
||||
add_executable(test_kdt test_kdt.cpp ${kdt_srcs})
|
||||
endif()
|
||||
target_link_libraries(test_kdt
|
||||
SPTAGLibStatic
|
||||
${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
#<<<<<<<<<<
|
||||
|
||||
|
||||
add_subdirectory(faiss_ori)
|
||||
add_subdirectory(test_nsg)
|
||||
|
||||
@ -1,15 +1,15 @@
|
||||
include_directories(${CMAKE_SOURCE_DIR}/thirdparty)
|
||||
include_directories(${CMAKE_SOURCE_DIR}/include)
|
||||
include_directories(${CORE_SOURCE_DIR}/thirdparty)
|
||||
include_directories(${CORE_SOURCE_DIR}/include)
|
||||
include_directories(/usr/local/cuda/include)
|
||||
link_directories(/usr/local/cuda/lib64)
|
||||
link_directories(${CMAKE_SOURCE_DIR}/thirdparty/tbb)
|
||||
link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb)
|
||||
|
||||
set(unittest_libs
|
||||
gtest gmock gtest_main gmock_main)
|
||||
|
||||
set(depend_libs
|
||||
faiss openblas lapack
|
||||
arrow jemalloc_pic
|
||||
arrow ${ARROW_PREFIX}/lib/libjemalloc_pic.a
|
||||
tbb
|
||||
)
|
||||
|
||||
@ -20,7 +20,7 @@ set(basic_libs
|
||||
|
||||
|
||||
#<GPU-TEST>
|
||||
#<<<<<<<<<<
|
||||
add_executable(test_gpu gpuresource_test.cpp)
|
||||
if(NOT TARGET test_gpu)
|
||||
add_executable(test_gpu gpuresource_test.cpp)
|
||||
endif()
|
||||
target_link_libraries(test_gpu ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
#<<<<<<<<<<
|
||||
|
||||
@ -26,6 +26,11 @@ class IDMAPTest : public DataGen, public ::testing::Test {
|
||||
Init_with_default();
|
||||
index_ = std::make_shared<IDMAP>();
|
||||
}
|
||||
|
||||
void TearDown() override {
|
||||
FaissGpuResourceMgr::GetInstance().Free();
|
||||
}
|
||||
|
||||
protected:
|
||||
IDMAPPtr index_ = nullptr;
|
||||
};
|
||||
|
||||
@ -7,13 +7,11 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <thread>
|
||||
|
||||
#include <faiss/AutoTune.h>
|
||||
#include <faiss/gpu/GpuAutoTune.h>
|
||||
#include <faiss/gpu/GpuIndexIVFFlat.h>
|
||||
#include <faiss/gpu/GpuClonerOptions.h>
|
||||
|
||||
#include "knowhere/index/vector_index/gpu_ivf.h"
|
||||
#include "knowhere/index/vector_index/ivf.h"
|
||||
@ -58,6 +56,9 @@ class IVFTest
|
||||
index_ = IndexFactory(index_type);
|
||||
FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024*1024*200, 1024*1024*300, 2);
|
||||
}
|
||||
void TearDown() override {
|
||||
FaissGpuResourceMgr::GetInstance().Free();
|
||||
}
|
||||
|
||||
protected:
|
||||
std::string index_type;
|
||||
@ -369,6 +370,7 @@ class GPURESTEST
|
||||
void TearDown() override {
|
||||
delete ids;
|
||||
delete dis;
|
||||
FaissGpuResourceMgr::GetInstance().Free();
|
||||
}
|
||||
|
||||
protected:
|
||||
|
||||
@ -2,7 +2,6 @@
|
||||
include_directories(/usr/local/include/gperftools)
|
||||
link_directories(/usr/local/lib)
|
||||
|
||||
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free")
|
||||
add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO)
|
||||
|
||||
find_package(OpenMP)
|
||||
@ -14,37 +13,29 @@ else ()
|
||||
endif ()
|
||||
message(${OpenMP_CXX_FLAGS})
|
||||
|
||||
include_directories(${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg)
|
||||
#include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/include/python3.6m)
|
||||
#include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/envs/megasearch_testframework_dev/lib/python3.6/site-packages/numpy/core/include)
|
||||
include_directories(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg)
|
||||
|
||||
aux_source_directory(${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src)
|
||||
|
||||
#add_library(nsg_raw SHARED ${nsg_src})
|
||||
#target_link_libraries(nsg_raw
|
||||
# gomp)
|
||||
aux_source_directory(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src)
|
||||
|
||||
set(interface_src
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
|
||||
../utils.cpp
|
||||
)
|
||||
|
||||
add_executable(test_nsg
|
||||
test_nsg.cpp
|
||||
${interface_src}
|
||||
${nsg_src}
|
||||
)
|
||||
#target_link_libraries(test_nsg
|
||||
# # libprofiler.so
|
||||
## -ltcmalloc
|
||||
## gomp
|
||||
# )
|
||||
if(NOT TARGET test_nsg)
|
||||
add_executable(test_nsg
|
||||
test_nsg.cpp
|
||||
${interface_src}
|
||||
${nsg_src}
|
||||
)
|
||||
endif()
|
||||
|
||||
target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
##############################
|
||||
@ -19,6 +19,7 @@
|
||||
#include <unistd.h>
|
||||
#include <string.h>
|
||||
#include <src/scheduler/SchedInst.h>
|
||||
#include "knowhere/index/vector_index/gpu_ivf.h"
|
||||
|
||||
#include "metrics/Metrics.h"
|
||||
#include "DBWrapper.h"
|
||||
@ -232,6 +233,7 @@ Server::StopService() {
|
||||
grpc::GrpcMilvusServer::StopService();
|
||||
DBWrapper::GetInstance().StopService();
|
||||
engine::StopSchedulerService();
|
||||
knowhere::FaissGpuResourceMgr::GetInstance().Free(); // free gpu resource.
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -952,15 +952,8 @@ DropIndexTask::OnExecute() {
|
||||
return SetError(res, "Invalid table name: " + table_name_);
|
||||
}
|
||||
|
||||
//step 2:check index existence
|
||||
engine::TableIndex index;
|
||||
engine::Status stat = DBWrapper::DB()->DescribeIndex(table_name_, index);
|
||||
if (index.engine_type_ == 1) {
|
||||
return SetError(SERVER_UNEXPECTED_ERROR, "index not existed");
|
||||
}
|
||||
|
||||
//step 3: check table existence
|
||||
stat = DBWrapper::DB()->DropIndex(table_name_);
|
||||
//step 2: check table existence
|
||||
auto stat = DBWrapper::DB()->DropIndex(table_name_);
|
||||
if (!stat.ok()) {
|
||||
return SetError(DB_META_TRANSACTION_FAILED, stat.ToString());
|
||||
}
|
||||
|
||||
@ -72,8 +72,11 @@ server::KnowhereError VecIndexImpl::Search(const long &nq, const float *xq, floa
|
||||
auto k = cfg["k"].as<int>();
|
||||
auto dataset = GenDataset(nq, dim, xq);
|
||||
|
||||
Config search_cfg;
|
||||
auto res = index_->Search(dataset, cfg);
|
||||
Config search_cfg = cfg;
|
||||
|
||||
ParameterValidation(type, search_cfg);
|
||||
|
||||
auto res = index_->Search(dataset, search_cfg);
|
||||
auto ids_array = res->array()[0];
|
||||
auto dis_array = res->array()[1];
|
||||
|
||||
|
||||
@ -71,7 +71,7 @@ size_t FileIOWriter::operator()(void *ptr, size_t size) {
|
||||
}
|
||||
|
||||
|
||||
VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config& cfg) {
|
||||
VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config &cfg) {
|
||||
std::shared_ptr<zilliz::knowhere::VectorIndex> index;
|
||||
auto gpu_device = cfg.get_with_default("gpu_id", 0);
|
||||
switch (type) {
|
||||
@ -235,6 +235,31 @@ void AutoGenParams(const IndexType &type, const long &size, zilliz::knowhere::Co
|
||||
}
|
||||
}
|
||||
|
||||
#if CUDA_VERSION > 9000
|
||||
#define GPU_MAX_NRPOBE 2048
|
||||
#else
|
||||
#define GPU_MAX_NRPOBE 1024
|
||||
#endif
|
||||
|
||||
void ParameterValidation(const IndexType &type, Config &cfg) {
|
||||
switch (type) {
|
||||
case IndexType::FAISS_IVFSQ8_GPU:
|
||||
case IndexType::FAISS_IVFFLAT_GPU:
|
||||
case IndexType::FAISS_IVFPQ_GPU: {
|
||||
if (cfg.get_with_default("nprobe", 0) != 0) {
|
||||
auto nprobe = cfg["nprobe"].as<int>();
|
||||
if (nprobe > GPU_MAX_NRPOBE) {
|
||||
WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE << ", but you passed " << nprobe
|
||||
<< ". Search with " << GPU_MAX_NRPOBE << " instead";
|
||||
cfg.insert_or_assign("nprobe", GPU_MAX_NRPOBE);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:break;
|
||||
}
|
||||
}
|
||||
|
||||
IndexType ConvertToCpuIndexType(const IndexType &type) {
|
||||
// TODO(linxj): add IDMAP
|
||||
switch (type) {
|
||||
|
||||
@ -14,6 +14,8 @@
|
||||
#include "knowhere/common/config.h"
|
||||
#include "knowhere/common/binary_set.h"
|
||||
|
||||
#include "cuda.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace milvus {
|
||||
@ -90,6 +92,8 @@ extern VecIndexPtr LoadVecIndex(const IndexType &index_type, const zilliz::knowh
|
||||
|
||||
extern void AutoGenParams(const IndexType& type, const long& size, Config& cfg);
|
||||
|
||||
extern void ParameterValidation(const IndexType& type, Config& cfg);
|
||||
|
||||
extern IndexType ConvertToCpuIndexType(const IndexType& type);
|
||||
extern IndexType ConvertToGpuIndexType(const IndexType& type);
|
||||
|
||||
|
||||
@ -38,9 +38,13 @@ set(unittest_libs
|
||||
cudart
|
||||
)
|
||||
|
||||
foreach(dir ${CORE_INCLUDE_DIRS})
|
||||
include_directories(${dir})
|
||||
endforeach()
|
||||
|
||||
add_subdirectory(server)
|
||||
add_subdirectory(db)
|
||||
#add_subdirectory(knowhere)
|
||||
add_subdirectory(knowhere)
|
||||
add_subdirectory(metrics)
|
||||
#add_subdirectory(scheduler)
|
||||
#add_subdirectory(storage)
|
||||
@ -35,8 +35,6 @@ link_directories("/usr/local/cuda/lib64")
|
||||
|
||||
include_directories(/usr/include/mysql)
|
||||
|
||||
#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY)
|
||||
|
||||
set(db_test_src
|
||||
${config_files}
|
||||
${cache_srcs}
|
||||
@ -69,25 +67,10 @@ set(db_libs
|
||||
|
||||
set(knowhere_libs
|
||||
knowhere
|
||||
SPTAGLibStatic
|
||||
arrow
|
||||
jemalloc_pic
|
||||
faiss
|
||||
openblas
|
||||
lapack
|
||||
tbb
|
||||
cudart
|
||||
cublas
|
||||
)
|
||||
|
||||
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
|
||||
set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS})
|
||||
else()
|
||||
set(db_libs ${db_libs}
|
||||
lapack
|
||||
openblas)
|
||||
endif()
|
||||
|
||||
target_link_libraries(db_test ${db_libs} ${knowhere_libs} ${unittest_libs})
|
||||
|
||||
install(TARGETS db_test DESTINATION unittest)
|
||||
|
||||
@ -8,13 +8,6 @@ set(helper
|
||||
|
||||
set(knowhere_libs
|
||||
knowhere
|
||||
SPTAGLibStatic
|
||||
arrow
|
||||
jemalloc_pic
|
||||
faiss
|
||||
openblas
|
||||
lapack
|
||||
tbb
|
||||
cudart
|
||||
cublas
|
||||
)
|
||||
|
||||
@ -40,6 +40,9 @@ class KnowhereWrapperTest
|
||||
|
||||
index_ = GetVecIndexFactory(index_type);
|
||||
}
|
||||
void TearDown() override {
|
||||
zilliz::knowhere::FaissGpuResourceMgr::GetInstance().Free();
|
||||
}
|
||||
|
||||
void AssertResult(const std::vector<long> &ids, const std::vector<float> &dis) {
|
||||
EXPECT_EQ(ids.size(), nq * k);
|
||||
|
||||
@ -1,13 +1,3 @@
|
||||
|
||||
|
||||
#set(CURL_LIBRARY "-lcurl")
|
||||
#find_package(CURL REQUIRED)
|
||||
#INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR})
|
||||
#
|
||||
#INCLUDE_DIRECTORIES(~/development/lib/usr/local/include)
|
||||
#LINK_DIRECTORIES(~/development/lib/usr/local/lib)
|
||||
|
||||
|
||||
include_directories(../../src)
|
||||
|
||||
|
||||
@ -72,13 +62,6 @@ add_executable(metrics_test ${count_test_src} ${require_files} )
|
||||
|
||||
set(knowhere_libs
|
||||
knowhere
|
||||
SPTAGLibStatic
|
||||
arrow
|
||||
jemalloc_pic
|
||||
faiss
|
||||
openblas
|
||||
lapack
|
||||
tbb
|
||||
)
|
||||
|
||||
target_link_libraries(metrics_test
|
||||
@ -96,12 +79,5 @@ target_link_libraries(metrics_test
|
||||
mysqlpp
|
||||
${unittest_libs}
|
||||
)
|
||||
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
|
||||
target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS})
|
||||
else()
|
||||
target_link_libraries(metrics_test
|
||||
lapack
|
||||
openblas)
|
||||
endif()
|
||||
|
||||
install(TARGETS metrics_test DESTINATION unittest)
|
||||
@ -33,8 +33,6 @@ link_directories("/usr/local/cuda/lib64")
|
||||
|
||||
include_directories(/usr/include/mysql)
|
||||
|
||||
#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY)
|
||||
|
||||
set(scheduler_test_src
|
||||
${unittest_srcs}
|
||||
${test_srcs}
|
||||
@ -69,25 +67,10 @@ set(scheduler_libs
|
||||
|
||||
set(knowhere_libs
|
||||
knowhere
|
||||
SPTAGLibStatic
|
||||
arrow
|
||||
jemalloc_pic
|
||||
faiss
|
||||
openblas
|
||||
lapack
|
||||
tbb
|
||||
cudart
|
||||
cublas
|
||||
)
|
||||
|
||||
if (${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
|
||||
set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS})
|
||||
else ()
|
||||
set(db_libs ${db_libs}
|
||||
lapack
|
||||
openblas)
|
||||
endif ()
|
||||
|
||||
|
||||
target_link_libraries(scheduler_test ${scheduler_libs} ${knowhere_libs} ${unittest_libs})
|
||||
|
||||
|
||||
@ -34,9 +34,6 @@ cuda_add_executable(server_test
|
||||
|
||||
set(require_libs
|
||||
knowhere
|
||||
faiss
|
||||
openblas
|
||||
lapack
|
||||
stdc++
|
||||
cudart
|
||||
cublas
|
||||
@ -51,14 +48,6 @@ set(require_libs
|
||||
pthread
|
||||
)
|
||||
|
||||
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
|
||||
set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS})
|
||||
else()
|
||||
set(require_libs ${require_libs}
|
||||
lapack
|
||||
openblas)
|
||||
endif()
|
||||
|
||||
target_link_libraries(server_test
|
||||
${require_libs}
|
||||
${cuda_library}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user