From ae5f518f5289ba7a585cd2c07bfa93afb2a0d7e7 Mon Sep 17 00:00:00 2001 From: XuanYang-cn <51370125+XuanYang-cn@users.noreply.github.com> Date: Fri, 14 Aug 2020 15:43:21 +0800 Subject: [PATCH] Optimize yaml-cpp download and build process (#3252) * modify yaml Signed-off-by: yangxuan * config dependency altering Signed-off-by: yangxuan * modify format Signed-off-by: yangxuan Co-authored-by: yangxuan --- core/CMakeLists.txt | 3 + core/cmake/ThirdPartyPackages.cmake | 55 ----------------- core/src/config/CMakeLists.txt | 4 +- core/thirdparty/CMakeLists.txt | 34 ++++++----- core/thirdparty/grpc/CMakeLists.txt | 78 ++++++++++++++----------- core/thirdparty/gtest/CMakeLists.txt | 4 +- core/thirdparty/versions.txt | 2 +- core/thirdparty/yaml-cpp/CMakeLists.txt | 50 ++++++++++++++++ 8 files changed, 120 insertions(+), 110 deletions(-) create mode 100644 core/thirdparty/yaml-cpp/CMakeLists.txt diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index e8ade37f86..5a1157f8e9 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -61,6 +61,9 @@ import_mysql_inc() using_ccache_if_defined( MILVUS_USE_CCACHE ) include( ExternalProject ) +include( FetchContent ) +set( FETCHCONTENT_BASE_DIR ${MILVUS_BINARY_DIR}/3rdparty_download ) +set(FETCHCONTENT_QUIET OFF) include( ThirdPartyPackages ) diff --git a/core/cmake/ThirdPartyPackages.cmake b/core/cmake/ThirdPartyPackages.cmake index 25781677c4..dd1117d4f9 100644 --- a/core/cmake/ThirdPartyPackages.cmake +++ b/core/cmake/ThirdPartyPackages.cmake @@ -14,7 +14,6 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES MySQLPP Prometheus SQLite - yaml-cpp libunwind gperftools ZLIB @@ -39,8 +38,6 @@ macro(build_dependency DEPENDENCY_NAME) build_prometheus() elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite") build_sqlite() - elseif ("${DEPENDENCY_NAME}" STREQUAL "yaml-cpp") - build_yamlcpp() elseif ("${DEPENDENCY_NAME}" STREQUAL "libunwind") build_libunwind() elseif ("${DEPENDENCY_NAME}" STREQUAL "gperftools") @@ -245,13 +242,6 @@ else () "https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz") endif () -if (DEFINED ENV{MILVUS_YAMLCPP_URL}) - set(YAMLCPP_SOURCE_URL "$ENV{MILVUS_YAMLCPP_URL}") -else () - set(YAMLCPP_SOURCE_URL "https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz" - "https://gitee.com/quicksilver/yaml-cpp/repository/archive/yaml-cpp-${YAMLCPP_VERSION}.zip") -endif () - if (DEFINED ENV{MILVUS_LIBUNWIND_URL}) set(LIBUNWIND_SOURCE_URL "$ENV{MILVUS_LIBUNWIND_URL}") else () @@ -492,51 +482,6 @@ if (MILVUS_WITH_SQLITE) link_directories(SYSTEM ${SQLITE_PREFIX}/lib/) endif () -# ---------------------------------------------------------------------- -# yaml-cpp - -macro(build_yamlcpp) - message(STATUS "Building yaml-cpp-${YAMLCPP_VERSION} from source") - set(YAMLCPP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/yaml-cpp_ep-prefix/src/yaml-cpp_ep") - set(YAMLCPP_STATIC_LIB "${YAMLCPP_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}yaml-cpp${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(YAMLCPP_INCLUDE_DIR "${YAMLCPP_PREFIX}/include") - set(YAMLCPP_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${YAMLCPP_PREFIX}" - -DCMAKE_INSTALL_LIBDIR=lib - -DYAML_CPP_BUILD_TESTS=OFF - -DYAML_CPP_BUILD_TOOLS=OFF) - - ExternalProject_Add(yaml-cpp_ep - URL - ${YAMLCPP_SOURCE_URL} - ${EP_LOG_OPTIONS} - URL_MD5 - "5b943e9af0060d0811148b037449ef82" - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - "${YAMLCPP_STATIC_LIB}" - CMAKE_ARGS - ${YAMLCPP_CMAKE_ARGS}) - - file(MAKE_DIRECTORY "${YAMLCPP_INCLUDE_DIR}") - add_library(yaml-cpp STATIC IMPORTED) - set_target_properties(yaml-cpp - PROPERTIES IMPORTED_LOCATION "${YAMLCPP_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${YAMLCPP_INCLUDE_DIR}") - - add_dependencies(yaml-cpp yaml-cpp_ep) -endmacro() - -if (MILVUS_WITH_YAMLCPP) - resolve_dependency(yaml-cpp) - - get_target_property(YAMLCPP_INCLUDE_DIR yaml-cpp INTERFACE_INCLUDE_DIRECTORIES) - link_directories(SYSTEM ${YAMLCPP_PREFIX}/lib/) - include_directories(SYSTEM ${YAMLCPP_INCLUDE_DIR}) -endif () # ---------------------------------------------------------------------- # libunwind diff --git a/core/src/config/CMakeLists.txt b/core/src/config/CMakeLists.txt index 018fa726dd..a9db68a410 100644 --- a/core/src/config/CMakeLists.txt +++ b/core/src/config/CMakeLists.txt @@ -21,7 +21,7 @@ set( CONFIG_SRCS ConfigMgr.h ) add_library( config ${CONFIG_SRCS} ) -add_dependencies( config yaml-cpp ) +target_link_libraries ( config yaml-cpp ) if ( BUILD_UNIT_TEST ) @@ -34,7 +34,7 @@ if ( BUILD_UNIT_TEST ) gtest_main gmock gmock_main - yaml-cpp ) + ) add_test ( NAME ConfigTypeTest COMMAND $ diff --git a/core/thirdparty/CMakeLists.txt b/core/thirdparty/CMakeLists.txt index bb3a757d8a..78bdffcb5a 100644 --- a/core/thirdparty/CMakeLists.txt +++ b/core/thirdparty/CMakeLists.txt @@ -14,10 +14,10 @@ # Thirdpart cxx and c flags add_compile_options( -O3 -fPIC -Wno-error -fopenmp ) -if (NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD) - set(EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1) +if ( NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD ) + set( EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1 ) else () - set(EP_LOG_OPTIONS) + set( EP_LOG_OPTIONS ) endif () set( MAKE_BUILD_ARGS "-j6" ) @@ -29,14 +29,14 @@ set( FETCHCONTENT_QUIET OFF ) # ---------------------------------------------------------------------- # Find pthreads -set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(Threads REQUIRED) +set( THREADS_PREFER_PTHREAD_FLAG ON ) +find_package( Threads REQUIRED ) # ****************************** Thirdparty googletest *************************************** -if (MILVUS_BUILD_TESTS) +if ( MILVUS_BUILD_TESTS ) if ( DEFINED ENV{MILVUS_GTEST_URL} ) set( GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}" ) - else () + else() set( GTEST_SOURCE_URL "https://gitee.com/quicksilver/googletest/repository/archive/release-${GTEST_VERSION}.zip" ) endif() @@ -44,13 +44,17 @@ if (MILVUS_BUILD_TESTS) endif() # ****************************** Thirdparty grpc *************************************** -if (MILVUS_WITH_GRPC) - if (DEFINED ENV{MILVUS_GRPC_URL}) - set(GRPC_SOURCE_URL "$ENV{MILVUS_GRPC_URL}") - else () - set(GRPC_SOURCE_URL - "https://github.com/milvus-io/grpc-milvus/archive/${GRPC_VERSION}.zip" - ) - endif () +if ( MILVUS_WITH_GRPC ) + if ( DEFINED ENV{MILVUS_GRPC_URL} ) + set( GRPC_SOURCE_URL "$ENV{MILVUS_GRPC_URL}" ) + else() + set( GRPC_SOURCE_URL + "https://github.com/milvus-io/grpc-milvus/archive/${GRPC_VERSION}.zip" ) + endif() add_subdirectory( grpc ) endif () + +# ****************************** Thirdparty yaml *************************************** +if ( MILVUS_WITH_YAMLCPP ) + add_subdirectory( yaml-cpp ) +endif() diff --git a/core/thirdparty/grpc/CMakeLists.txt b/core/thirdparty/grpc/CMakeLists.txt index 7d939cef43..3bcd2aa11f 100644 --- a/core/thirdparty/grpc/CMakeLists.txt +++ b/core/thirdparty/grpc/CMakeLists.txt @@ -1,23 +1,29 @@ -# Another way is to use CMake's FetchContent module to clone gRPC at -# configure time. This makes gRPC's source code available to project, -message(STATUS "Building GRPC-${GRPC_VERSION} from source") -# ****************************************************************** -# FetchContent -# ****************************************************************** +#------------------------------------------------------------------------------- +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. +#------------------------------------------------------------------------------- + +message( STATUS "Building GRPC-${GRPC_VERSION} from source" ) FetchContent_Declare( grpc - URL ${GRPC_SOURCE_URL} - URL_MD5 "478215c151a144c2d8625b49ff1b70aa" - SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc-src - BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc-build - ) -set(FETCHCONTENT_QUIET OFF) + URL ${GRPC_SOURCE_URL} + URL_MD5 "478215c151a144c2d8625b49ff1b70aa" + SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc-src + BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc-build ) # using openSSL rather than boringssl set(gRPC_SSL_PROVIDER "package" CACHE STRING "OPENSSL" FORCE) FetchContent_GetProperties( grpc ) -if ( NOT grpc_POPULATED ) +if ( NOT grpc_POPULATED ) FetchContent_Populate( grpc ) # Adding the following targets: @@ -27,13 +33,12 @@ if ( NOT grpc_POPULATED ) # grpc++ add_subdirectory( ${grpc_SOURCE_DIR} ${grpc_BINARY_DIR} - EXCLUDE_FROM_ALL - ) + EXCLUDE_FROM_ALL ) endif() # get grpc COMPILE_OPTIONS ) -get_property(var DIRECTORY "${grpc_SOURCE_DIR}" PROPERTY COMPILE_OPTIONS ) -message(STATUS "grpc src compile options: ${var}") +get_property( var DIRECTORY "${grpc_SOURCE_DIR}" PROPERTY COMPILE_OPTIONS ) +message( STATUS "grpc src compile options: ${var}" ) # ************************************************************************** @@ -53,36 +58,39 @@ get_filename_component( status_proto_path "${PROTO_PATH}" PATH ) # Generated sources -set(milvus_proto_srcs "${PROTO_PATH}/gen-milvus-test/helloworld.pb.cc") -set(milvus_proto_hdrs "${PROTO_PATH}/gen-milvus-test/helloworld.pb.h") -set(milvus_grpc_srcs "${PROTO_PATH}/gen-milvus-test/helloworld.grpc.pb.cc") -set(milvus_grpc_hdrs "${PROTO_PATH}/gen-milvus-test/helloworld.grpc.pb.h") +set( milvus_proto_srcs "${PROTO_PATH}/gen-milvus-test/helloworld.pb.cc" ) +set( milvus_proto_hdrs "${PROTO_PATH}/gen-milvus-test/helloworld.pb.h" ) +set( milvus_grpc_srcs "${PROTO_PATH}/gen-milvus-test/helloworld.grpc.pb.cc" ) +set( milvus_grpc_hdrs "${PROTO_PATH}/gen-milvus-test/helloworld.grpc.pb.h" ) -set(status_proto_srcs "${PROTO_PATH}/gen-status-test/helloworld.pb.cc") -set(status_proto_hdrs "${PROTO_PATH}/gen-status-test/helloworld.pb.h") -set(status_grpc_srcs "${PROTO_PATH}/gen-status-test/helloworld.grpc.pb.cc") -set(status_grpc_hdrs "${PROTO_PATH}/gen-status-test/helloworld.grpc.pb.h") +set( status_proto_srcs "${PROTO_PATH}/gen-status-test/helloworld.pb.cc" ) +set( status_proto_hdrs "${PROTO_PATH}/gen-status-test/helloworld.pb.h" ) +set( status_grpc_srcs "${PROTO_PATH}/gen-status-test/helloworld.grpc.pb.cc" ) +set( status_grpc_hdrs "${PROTO_PATH}/gen-status-test/helloworld.grpc.pb.h" ) add_custom_command( - OUTPUT "${milvus_proto_srcs}" "${milvus_proto_hdrs}" "${milvus_grpc_srcs}" "${milvus_grpc_hdrs}" - COMMAND ${PROTOC_EXCUTABLE} - ARGS --grpc_out "${PROTO_PATH}/gen-milvus-test" + OUTPUT "${milvus_proto_srcs}" + "${milvus_proto_hdrs}" + "${milvus_grpc_srcs}" + "${milvus_grpc_hdrs}" + COMMAND ${PROTOC_EXCUTABLE} + ARGS --grpc_out "${PROTO_PATH}/gen-milvus-test" --cpp_out "${PROTO_PATH}/gen-milvus-test" -I "${milvus_proto_path}" --plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN_EXCUTABLE}" "${milvus_proto}" - DEPENDS "${milvus_proto}" ) + DEPENDS "${milvus_proto}" ) add_custom_command( - OUTPUT "${status_proto_srcs}" "${status_proto_hdrs}" "${status_grpc_srcs}" "${status_grpc_hdrs}" - COMMAND ${PROTOC_EXCUTABLE} - ARGS --grpc_out "${PROTO_PATH}/gen-status-test" + OUTPUT "${status_proto_srcs}" "${status_proto_hdrs}" "${status_grpc_srcs}" "${status_grpc_hdrs}" + COMMAND ${PROTOC_EXCUTABLE} + ARGS --grpc_out "${PROTO_PATH}/gen-status-test" --cpp_out "${PROTO_PATH}/gen-status-test" -I "${status_proto_path}" --plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN_EXCUTABLE}" "${status_proto}" - DEPENDS "${status_proto}" ) + DEPENDS "${status_proto}" ) # Include generated *.pb.h files -include_directories("${MILVUS_SOURCE_DIR}/gen-milvus-test") -include_directories("${MILVUS_SOURCE_DIR}/gen-status-test") +include_directories( "${MILVUS_SOURCE_DIR}/gen-milvus-test") +include_directories( "${MILVUS_SOURCE_DIR}/gen-status-test" ) diff --git a/core/thirdparty/gtest/CMakeLists.txt b/core/thirdparty/gtest/CMakeLists.txt index 47558c170e..678b35eb29 100644 --- a/core/thirdparty/gtest/CMakeLists.txt +++ b/core/thirdparty/gtest/CMakeLists.txt @@ -55,5 +55,5 @@ endif() # endif() -get_property(var DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/googletest-src" PROPERTY COMPILE_OPTIONS ) -message(STATUS "gtest compile options: ${var}") +get_property( var DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/googletest-src" PROPERTY COMPILE_OPTIONS ) +message( STATUS "gtest compile options: ${var}" ) diff --git a/core/thirdparty/versions.txt b/core/thirdparty/versions.txt index ac8c40cd11..e9a0ad37b0 100644 --- a/core/thirdparty/versions.txt +++ b/core/thirdparty/versions.txt @@ -3,7 +3,7 @@ GTEST_VERSION=1.8.1 MYSQLPP_VERSION=3.2.4 PROMETHEUS_VERSION=0.7.0 SQLITE_VERSION=3280000 -YAMLCPP_VERSION=0.6.2 +YAMLCPP_VERSION=0.6.3 LIBUNWIND_VERSION=1.3.1 GPERFTOOLS_VERSION=2.7 GRPC_VERSION=4b1c02a7edd03a2fec7dec48f37186306c456378 diff --git a/core/thirdparty/yaml-cpp/CMakeLists.txt b/core/thirdparty/yaml-cpp/CMakeLists.txt new file mode 100644 index 0000000000..8b36cae013 --- /dev/null +++ b/core/thirdparty/yaml-cpp/CMakeLists.txt @@ -0,0 +1,50 @@ +#------------------------------------------------------------------------------- +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. +#------------------------------------------------------------------------------- + +if ( DEFINED ENV{MILVUS_YAMLCPP_URL} ) + set( YAMLCPP_SOURCE_URL "$ENV{MILVUS_YAMLCPP_URL}" ) +else() + set( YAMLCPP_SOURCE_URL + "https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz" + "https://gitee.com/quicksilver/yaml-cpp/repository/archive/yaml-cpp-${YAMLCPP_VERSION}.zip" ) +endif() + +message( STATUS "Building yaml-cpp-${YAMLCPP_VERSION} from source" ) +FetchContent_Declare( + yaml-cpp + URL ${YAMLCPP_SOURCE_URL} + URL_MD5 "b45bf1089a382e81f6b661062c10d0c2" + SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/yaml-src + BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/yaml-build + ) + +set( YAML_CPP_BUILD_TESTS CACHE BOOL OFF FORCE ) +set( YAML_CPP_BUILD_TOOLS CACHE BOOL OFF FORCE ) +FetchContent_GetProperties( yaml-cpp ) +if ( NOT yaml-cpp_POPULATED ) + + FetchContent_Populate( yaml-cpp ) + + # Adding the following targets: + # yaml-cpp::yaml-cpp, yaml-cpp + add_subdirectory( ${yaml-cpp_SOURCE_DIR} + ${yaml-cpp_BINARY_DIR} + EXCLUDE_FROM_ALL ) + +endif() + +get_target_property( YAML_CPP_INCLUDE_DIR yaml-cpp INTERFACE_INCLUDE_DIRECTORIES ) +message( STATUS ${YAML_CPP_INCLUDE_DIR} ) + +get_property( var DIRECTORY "${yaml-cpp_SOURCE_DIR}" PROPERTY COMPILE_OPTIONS ) +message( STATUS "yaml compile options: ${var}" )