From 157f24b4540653bfaa4914392bd09fdc0ddb1c03 Mon Sep 17 00:00:00 2001 From: XuanYang-cn <51370125+XuanYang-cn@users.noreply.github.com> Date: Thu, 20 Aug 2020 12:46:46 +0800 Subject: [PATCH] Download path can be configured (#3350) * download path configuration Signed-off-by: yangxuan * add -f to build.sh Signed-off-by: yangxuan * change mysql cxx_flags to aviod print Wdeprecated warnings Signed-off-by: yangxuan Co-authored-by: yangxuan --- core/build.sh | 10 ++++++++-- core/thirdparty/CMakeLists.txt | 7 +++++++ core/thirdparty/crc32c/CMakeLists.txt | 11 ++++++----- core/thirdparty/grpc/CMakeLists.txt | 20 +++++++++----------- core/thirdparty/gtest/CMakeLists.txt | 21 ++++++++++----------- core/thirdparty/mysqlpp/CMakeLists.txt | 8 ++++---- core/thirdparty/opentracing/CMakeLists.txt | 22 +++++++++++----------- core/thirdparty/profilers/CMakeLists.txt | 2 +- core/thirdparty/yaml-cpp/CMakeLists.txt | 13 ++++++------- 9 files changed, 62 insertions(+), 52 deletions(-) diff --git a/core/build.sh b/core/build.sh index 9df4f165fc..ba18c1a96d 100755 --- a/core/build.sh +++ b/core/build.sh @@ -18,9 +18,13 @@ CUDA_COMPILER=/usr/local/cuda/bin/nvcc GPU_VERSION="OFF" #defaults to CPU version WITH_PROMETHEUS="ON" CUDA_ARCH="DEFAULT" +CUSTOM_THIRDPARTY_PATH="" -while getopts "p:d:t:s:ulrcghzme" arg; do +while getopts "p:d:t:s:f:ulrcghzme" arg; do case $arg in + f) + CUSTOM_THIRDPARTY_PATH=$OPTARG + ;; p) INSTALL_PREFIX=$OPTARG ;; @@ -61,6 +65,7 @@ while getopts "p:d:t:s:ulrcghzme" arg; do echo " parameter: +-f: custom paths of thirdparty downloaded files(default: NULL) -p: install prefix(default: $(pwd)/milvus) -d: db data path(default: /tmp/milvus) -t: build type(default: Debug) @@ -75,7 +80,7 @@ parameter: -h: help usage: -./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -s \${CUDA_ARCH}[-u] [-l] [-r] [-c] [-z] [-g] [-m] [-e] [-h] +./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -s \${CUDA_ARCH} -f\${CUSTOM_THIRDPARTY_PATH} [-u] [-l] [-r] [-c] [-z] [-g] [-m] [-e] [-h] " exit 0 ;; @@ -115,6 +120,7 @@ CMAKE_CMD="cmake \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ -DMILVUS_WITH_PROMETHEUS=${WITH_PROMETHEUS} \ -DMILVUS_CUDA_ARCH=${CUDA_ARCH} \ +-DCUSTOM_THIRDPARTY_DOWNLOAD_PATH=${CUSTOM_THIRDPARTY_PATH} \ ../" echo ${CMAKE_CMD} ${CMAKE_CMD} diff --git a/core/thirdparty/CMakeLists.txt b/core/thirdparty/CMakeLists.txt index 50d512f612..f550e763f1 100644 --- a/core/thirdparty/CMakeLists.txt +++ b/core/thirdparty/CMakeLists.txt @@ -26,6 +26,13 @@ include( FetchContent ) set( FETCHCONTENT_BASE_DIR ${MILVUS_BINARY_DIR}/3rdparty_download ) set( FETCHCONTENT_QUIET OFF ) +if( CUSTOM_THIRDPARTY_DOWNLOAD_PATH ) + set( THIRDPARTY_DOWNLOAD_PATH ${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} ) +else() + set( THIRDPARTY_DOWNLOAD_PATH ${CMAKE_BINARY_DIR}/3rdparty_download/download ) +endif() +message( STATUS "Thirdparty downloaded file path: ${THIRDPARTY_DOWNLOAD_PATH}" ) + # ---------------------------------------------------------------------- # Find pthreads diff --git a/core/thirdparty/crc32c/CMakeLists.txt b/core/thirdparty/crc32c/CMakeLists.txt index a2d52149ad..d4b153e795 100644 --- a/core/thirdparty/crc32c/CMakeLists.txt +++ b/core/thirdparty/crc32c/CMakeLists.txt @@ -20,16 +20,17 @@ endif() message( STATUS "Building crc32c-master from source" ) FetchContent_Declare( crc32c - URL ${CRC32C_SOURCE_URL} - URL_MD5 "d24bd57ca2d9fb051aa589818c0e2c10" - SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/crc32c-src - BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/crc32c-build -) + URL ${CRC32C_SOURCE_URL} + URL_MD5 "d24bd57ca2d9fb051aa589818c0e2c10" + SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/crc32c-src + BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/crc32c-build + DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} ) FetchContent_GetProperties( crc32c ) if ( NOT crc32c_POPULATED ) FetchContent_Populate( crc32c ) + set( CRC32C_BUILD_TESTS CACHE BOOL OFF FORCE ) set( CRC32C_BUILD_BENCHMARKS CACHE BOOL OFF FORCE ) set( CRC32C_USE_GLOG CACHE BOOL OFF FORCE ) diff --git a/core/thirdparty/grpc/CMakeLists.txt b/core/thirdparty/grpc/CMakeLists.txt index 015f0ac0d0..a702a62aee 100644 --- a/core/thirdparty/grpc/CMakeLists.txt +++ b/core/thirdparty/grpc/CMakeLists.txt @@ -14,24 +14,22 @@ if ( DEFINED ENV{MILVUS_GRPC_URL} ) set( GRPC_SOURCE_URL "$ENV{MILVUS_GRPC_URL}" ) else() - set( GRPC_SOURCE_URL - "https://github.com/milvus-io/grpc-milvus/archive/${GRPC_VERSION}.zip" - ) + set( GRPC_SOURCE_URL "https://github.com/milvus-io/grpc-milvus/archive/${GRPC_VERSION}.zip" ) endif() message( STATUS "Building GRPC-${GRPC_VERSION} from source" ) FetchContent_Declare( - grpc - URL ${GRPC_SOURCE_URL} - URL_MD5 "f11a28bebdc14e7790dab64743a96836" - DOWNLOAD_DIR ${MILVUS_BINARY_DIR}/3rdparty_download/download - SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc-src - BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc-build ) + grpc + URL ${GRPC_SOURCE_URL} + URL_MD5 "f11a28bebdc14e7790dab64743a96836" + SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc-src + BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc-build + DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} ) # using openSSL rather than boringssl -set(gRPC_SSL_PROVIDER "package" CACHE STRING "OPENSSL" FORCE) -FetchContent_GetProperties( grpc ) +set( gRPC_SSL_PROVIDER "package" CACHE STRING "OPENSSL" FORCE ) +FetchContent_GetProperties( grpc ) if ( NOT grpc_POPULATED ) FetchContent_Populate( grpc ) diff --git a/core/thirdparty/gtest/CMakeLists.txt b/core/thirdparty/gtest/CMakeLists.txt index 139fbfc5ac..0e6b65313e 100644 --- a/core/thirdparty/gtest/CMakeLists.txt +++ b/core/thirdparty/gtest/CMakeLists.txt @@ -14,25 +14,24 @@ if ( DEFINED ENV{MILVUS_GTEST_URL} ) set( GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}" ) else() - set( GTEST_SOURCE_URL - "https://gitee.com/quicksilver/googletest/repository/archive/release-${GTEST_VERSION}.zip" ) + set( GTEST_SOURCE_URL "https://gitee.com/quicksilver/googletest/repository/archive/release-${GTEST_VERSION}.zip" ) endif() message( STATUS "Building gtest-${GTEST_VERSION} from source" ) -include( FetchContent ) + set( CMAKE_POLICY_DEFAULT_CMP0022 NEW ) # for googletest only FetchContent_Declare( - googletest - URL ${GTEST_SOURCE_URL} - URL_MD5 "f9137c5bc18b7d74027936f0f1bfa5c8" - DOWNLOAD_DIR ${MILVUS_BINARY_DIR}/3rdparty_download/download - SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest-src - BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest-build - -) + googletest + URL ${GTEST_SOURCE_URL} + URL_MD5 "f9137c5bc18b7d74027936f0f1bfa5c8" + SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest-src + BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest-build + DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} ) +FetchContent_GetProperties( googletest ) if ( NOT googletest_POPULATED ) + FetchContent_Populate( googletest ) # Adding the following targets: diff --git a/core/thirdparty/mysqlpp/CMakeLists.txt b/core/thirdparty/mysqlpp/CMakeLists.txt index 4e4873054b..54ebd2fc64 100644 --- a/core/thirdparty/mysqlpp/CMakeLists.txt +++ b/core/thirdparty/mysqlpp/CMakeLists.txt @@ -19,18 +19,18 @@ endif () macro( build_mysqlpp ) message( STATUS "Building MySQL++-${MYSQLPP_VERSION} from source" ) + set( MYSQLPP_CXX_FLAGS "-O3 -fPIC -Wno-deprecated -Wno-format-truncation -fopenmp" ) ExternalProject_Add( - mysqlpp_ep + mysqlpp_ep PREFIX ${CMAKE_BINARY_DIR}/3rdparty_download/mysqlpp-subbuild - DOWNLOAD_DIR ${CMAKE_BINARY_DIR}/3rdparty_download/download + DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR} URL ${MYSQLPP_SOURCE_URL} URL_MD5 "cda38b5ecc0117de91f7c42292dd1e79" CONFIGURE_COMMAND /configure "--prefix=" "--enable-thread-check" - "CFLAGS=${EP_C_FLAGS}" - "CXXFLAGS=${EP_CXX_FLAGS}" + "CXXFLAGS=${MYSQLPP_CXX_FLAGS}" "LDFLAGS=-pthread" "cc=${CCACHE_FOUND} ${CMAKE_C_COMPILER}" "cxx=${CCACHE_FOUND} ${CMAKE_CXX_COMPILER}" diff --git a/core/thirdparty/opentracing/CMakeLists.txt b/core/thirdparty/opentracing/CMakeLists.txt index 0cc05ee624..937479b121 100644 --- a/core/thirdparty/opentracing/CMakeLists.txt +++ b/core/thirdparty/opentracing/CMakeLists.txt @@ -12,31 +12,30 @@ #------------------------------------------------------------------------------- if ( DEFINED ENV{MILVUS_OPENTRACING_URL} ) - set(OPENTRACING_SOURCE_URL "$ENV{MILVUS_OPENTRACING_URL}") + set( OPENTRACING_SOURCE_URL "$ENV{MILVUS_OPENTRACING_URL}" ) else () - set(OPENTRACING_SOURCE_URL - "https://github.com/opentracing/opentracing-cpp/archive/${OPENTRACING_VERSION}.tar.gz" ) + set( OPENTRACING_SOURCE_URL + "https://github.com/opentracing/opentracing-cpp/archive/${OPENTRACING_VERSION}.tar.gz" ) endif () -message(STATUS "Building OPENTRACING-${OPENTRACING_VERSION} from source") +message( STATUS "Building OPENTRACING-${OPENTRACING_VERSION} from source" ) FetchContent_Declare( opentracing - URL ${OPENTRACING_SOURCE_URL} - URL_MD5 "e598ba4b81ae8e1ceed8cd8bbf86f2fd" - DOWNLOAD_DIR ${MILVUS_BINARY_DIR}/3rdparty_download/download + URL ${OPENTRACING_SOURCE_URL} + URL_MD5 "e598ba4b81ae8e1ceed8cd8bbf86f2fd" SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/opentracing-src BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/opentracing-build - ) + DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} ) -set( BUILD_SHARED_LIBS CACHE BOOL OFF FORCE) -set( ENABLE_LINTING CACHE BOOL OFF FORCE) -set( BUILD_TESTING CACHE BOOL OFF FORCE ) FetchContent_GetProperties( opentracing ) if ( NOT opentracing_POPULATED ) FetchContent_Populate( opentracing ) + set( BUILD_SHARED_LIBS CACHE BOOL OFF FORCE ) + set( ENABLE_LINTING CACHE BOOL OFF FORCE ) + set( BUILD_TESTING CACHE BOOL OFF FORCE ) # Adding the following targets: # opentracing-static # opentracing_mocktracer-static @@ -52,6 +51,7 @@ if ( NOT opentracing_POPULATED ) $ ) target_include_directories( opentracing_mocktracer-static INTERFACE $ ) + # Adding the following ALIAS Targets: # opentracing::opentracing # opentracing::mocktracer diff --git a/core/thirdparty/profilers/CMakeLists.txt b/core/thirdparty/profilers/CMakeLists.txt index cadafae379..cca8a9e7cd 100644 --- a/core/thirdparty/profilers/CMakeLists.txt +++ b/core/thirdparty/profilers/CMakeLists.txt @@ -34,7 +34,7 @@ macro( build_libunwind ) ExternalProject_Add( libunwind_ep PREFIX ${MILVUS_BINARY_DIR}/3rdparty_download/libunwind-subbuild - DOWNLOAD_DIR ${MILVUS_BINARY_DIR}/3rdparty_download/download + DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/libunwind URL ${LIBUNWIND_SOURCE_URL} URL_MD5 a04f69d66d8e16f8bf3ab72a69112cd6 diff --git a/core/thirdparty/yaml-cpp/CMakeLists.txt b/core/thirdparty/yaml-cpp/CMakeLists.txt index a21b7b751b..2bf4aa84e4 100644 --- a/core/thirdparty/yaml-cpp/CMakeLists.txt +++ b/core/thirdparty/yaml-cpp/CMakeLists.txt @@ -20,13 +20,12 @@ endif() message( STATUS "Building yaml-cpp-${YAMLCPP_VERSION} from source" ) FetchContent_Declare( - yaml-cpp - URL ${YAMLCPP_SOURCE_URL} - URL_MD5 "b45bf1089a382e81f6b661062c10d0c2" - DOWNLOAD_DIR ${MILVUS_BINARY_DIR}/3rdparty_download/download - SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/yaml-src - BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/yaml-build - ) + yaml-cpp + URL ${YAMLCPP_SOURCE_URL} + URL_MD5 "b45bf1089a382e81f6b661062c10d0c2" + SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/yaml-src + BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/yaml-build + DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} ) set( YAML_CPP_BUILD_TESTS CACHE BOOL OFF FORCE ) set( YAML_CPP_BUILD_TOOLS CACHE BOOL OFF FORCE )