Optimize yaml-cpp download and build process (#3252)

* modify yaml Signed-off-by: yangxuan <xuan.yang@zilliz.com> * config dependency altering Signed-off-by: yangxuan <xuan.yang@zilliz.com> * modify format Signed-off-by: yangxuan <xuan.yang@zilliz.com> Co-authored-by: yangxuan <xuan.yang@zilliz.com>
2026-02-02 01:06:41 +08:00 · 2020-08-14 15:43:21 +08:00 · 2020-08-14 15:43:21 +08:00 · ae5f518f52
commit ae5f518f52
parent 892b16633b
8 changed files with 120 additions and 110 deletions
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@ -61,6 +61,9 @@ import_mysql_inc()
 using_ccache_if_defined( MILVUS_USE_CCACHE )

 include( ExternalProject )
+include( FetchContent )
+set( FETCHCONTENT_BASE_DIR ${MILVUS_BINARY_DIR}/3rdparty_download )
+set(FETCHCONTENT_QUIET OFF)
 include( ThirdPartyPackages )


--- a/core/cmake/ThirdPartyPackages.cmake
+++ b/core/cmake/ThirdPartyPackages.cmake
@ -14,7 +14,6 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES
        MySQLPP
        Prometheus
        SQLite
-        yaml-cpp
        libunwind
        gperftools
        ZLIB
@ -39,8 +38,6 @@ macro(build_dependency DEPENDENCY_NAME)
        build_prometheus()
    elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite")
        build_sqlite()
-    elseif ("${DEPENDENCY_NAME}" STREQUAL "yaml-cpp")
-        build_yamlcpp()
    elseif ("${DEPENDENCY_NAME}" STREQUAL "libunwind")
        build_libunwind()
    elseif ("${DEPENDENCY_NAME}" STREQUAL "gperftools")
@ -245,13 +242,6 @@ else ()
            "https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz")
 endif ()

-if (DEFINED ENV{MILVUS_YAMLCPP_URL})
-    set(YAMLCPP_SOURCE_URL "$ENV{MILVUS_YAMLCPP_URL}")
-else ()
-    set(YAMLCPP_SOURCE_URL "https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz"
-                           "https://gitee.com/quicksilver/yaml-cpp/repository/archive/yaml-cpp-${YAMLCPP_VERSION}.zip")
-endif ()
-
 if (DEFINED ENV{MILVUS_LIBUNWIND_URL})
    set(LIBUNWIND_SOURCE_URL "$ENV{MILVUS_LIBUNWIND_URL}")
 else ()
@ -492,51 +482,6 @@ if (MILVUS_WITH_SQLITE)
    link_directories(SYSTEM ${SQLITE_PREFIX}/lib/)
 endif ()

-# ----------------------------------------------------------------------
-# yaml-cpp
-
-macro(build_yamlcpp)
-    message(STATUS "Building yaml-cpp-${YAMLCPP_VERSION} from source")
-    set(YAMLCPP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/yaml-cpp_ep-prefix/src/yaml-cpp_ep")
-    set(YAMLCPP_STATIC_LIB "${YAMLCPP_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}yaml-cpp${CMAKE_STATIC_LIBRARY_SUFFIX}")
-    set(YAMLCPP_INCLUDE_DIR "${YAMLCPP_PREFIX}/include")
-    set(YAMLCPP_CMAKE_ARGS
-            ${EP_COMMON_CMAKE_ARGS}
-            "-DCMAKE_INSTALL_PREFIX=${YAMLCPP_PREFIX}"
-            -DCMAKE_INSTALL_LIBDIR=lib
-            -DYAML_CPP_BUILD_TESTS=OFF
-            -DYAML_CPP_BUILD_TOOLS=OFF)
-
-    ExternalProject_Add(yaml-cpp_ep
-            URL
-            ${YAMLCPP_SOURCE_URL}
-            ${EP_LOG_OPTIONS}
-            URL_MD5
-            "5b943e9af0060d0811148b037449ef82"
-            BUILD_COMMAND
-            ${MAKE}
-            ${MAKE_BUILD_ARGS}
-            BUILD_BYPRODUCTS
-            "${YAMLCPP_STATIC_LIB}"
-            CMAKE_ARGS
-            ${YAMLCPP_CMAKE_ARGS})
-
-    file(MAKE_DIRECTORY "${YAMLCPP_INCLUDE_DIR}")
-    add_library(yaml-cpp STATIC IMPORTED)
-    set_target_properties(yaml-cpp
-            PROPERTIES IMPORTED_LOCATION "${YAMLCPP_STATIC_LIB}"
-            INTERFACE_INCLUDE_DIRECTORIES "${YAMLCPP_INCLUDE_DIR}")
-
-    add_dependencies(yaml-cpp yaml-cpp_ep)
-endmacro()
-
-if (MILVUS_WITH_YAMLCPP)
-    resolve_dependency(yaml-cpp)
-
-    get_target_property(YAMLCPP_INCLUDE_DIR yaml-cpp INTERFACE_INCLUDE_DIRECTORIES)
-    link_directories(SYSTEM ${YAMLCPP_PREFIX}/lib/)
-    include_directories(SYSTEM ${YAMLCPP_INCLUDE_DIR})
-endif ()

 # ----------------------------------------------------------------------
 # libunwind
--- a/core/src/config/CMakeLists.txt
+++ b/core/src/config/CMakeLists.txt
@ -21,7 +21,7 @@ set( CONFIG_SRCS    ConfigMgr.h
                    )

 add_library( config ${CONFIG_SRCS} )
-add_dependencies( config yaml-cpp )
+target_link_libraries ( config yaml-cpp )


 if ( BUILD_UNIT_TEST )
@ -34,7 +34,7 @@ if ( BUILD_UNIT_TEST )
                           gtest_main
                           gmock
                           gmock_main
-                           yaml-cpp )
+                           )

    add_test ( NAME     ConfigTypeTest
               COMMAND  $<TARGET_FILE:ConfigTypeTest>
--- a/core/thirdparty/CMakeLists.txt
+++ b/core/thirdparty/CMakeLists.txt
@ -14,10 +14,10 @@
 # Thirdpart cxx and c flags
 add_compile_options( -O3 -fPIC -Wno-error -fopenmp )

-if (NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD)
-    set(EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1)
+if ( NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD )
+    set( EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1 )
 else ()
-    set(EP_LOG_OPTIONS)
+    set( EP_LOG_OPTIONS )
 endif ()

 set( MAKE_BUILD_ARGS "-j6" )
@ -29,14 +29,14 @@ set( FETCHCONTENT_QUIET     OFF )
 # ----------------------------------------------------------------------
 # Find pthreads

-set(THREADS_PREFER_PTHREAD_FLAG ON)
-find_package(Threads REQUIRED)
+set( THREADS_PREFER_PTHREAD_FLAG ON )
+find_package( Threads REQUIRED )

 # ****************************** Thirdparty googletest ***************************************
-if (MILVUS_BUILD_TESTS)
+if ( MILVUS_BUILD_TESTS )
    if ( DEFINED ENV{MILVUS_GTEST_URL} )
        set( GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}" )
-    else ()
+    else()
        set( GTEST_SOURCE_URL
            "https://gitee.com/quicksilver/googletest/repository/archive/release-${GTEST_VERSION}.zip" )
    endif()
@ -44,13 +44,17 @@ if (MILVUS_BUILD_TESTS)
 endif()

 # ****************************** Thirdparty grpc ***************************************
-if (MILVUS_WITH_GRPC)
-    if (DEFINED ENV{MILVUS_GRPC_URL})
-        set(GRPC_SOURCE_URL "$ENV{MILVUS_GRPC_URL}")
-    else ()
-        set(GRPC_SOURCE_URL
-                "https://github.com/milvus-io/grpc-milvus/archive/${GRPC_VERSION}.zip"
-                )
-    endif ()
+if ( MILVUS_WITH_GRPC )
+    if ( DEFINED ENV{MILVUS_GRPC_URL} )
+        set( GRPC_SOURCE_URL "$ENV{MILVUS_GRPC_URL}" )
+    else()
+        set( GRPC_SOURCE_URL
+             "https://github.com/milvus-io/grpc-milvus/archive/${GRPC_VERSION}.zip" )
+    endif()
    add_subdirectory( grpc )
 endif ()
+
+# ****************************** Thirdparty yaml ***************************************
+if ( MILVUS_WITH_YAMLCPP )
+    add_subdirectory( yaml-cpp )
+endif()
--- a/core/thirdparty/grpc/CMakeLists.txt
+++ b/core/thirdparty/grpc/CMakeLists.txt
@ -1,23 +1,29 @@
-# Another way is to use CMake's FetchContent module to clone gRPC at
-# configure time. This makes gRPC's source code available to project,
-message(STATUS "Building GRPC-${GRPC_VERSION} from source")
-# ******************************************************************
-# FetchContent
-# ******************************************************************
+#-------------------------------------------------------------------------------
+# Copyright (C) 2019-2020 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+#-------------------------------------------------------------------------------
+
+message( STATUS "Building GRPC-${GRPC_VERSION} from source" )
 FetchContent_Declare(
    grpc
-    URL     ${GRPC_SOURCE_URL}
-    URL_MD5 "478215c151a144c2d8625b49ff1b70aa"
-    SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc-src
-    BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc-build
-    )
-set(FETCHCONTENT_QUIET OFF)
+    URL         ${GRPC_SOURCE_URL}
+    URL_MD5     "478215c151a144c2d8625b49ff1b70aa"
+    SOURCE_DIR  ${CMAKE_CURRENT_BINARY_DIR}/grpc-src
+    BINARY_DIR  ${CMAKE_CURRENT_BINARY_DIR}/grpc-build )

 # using openSSL rather than boringssl
 set(gRPC_SSL_PROVIDER "package" CACHE STRING "OPENSSL" FORCE)
 FetchContent_GetProperties( grpc )
-if ( NOT grpc_POPULATED )

+if ( NOT grpc_POPULATED )
    FetchContent_Populate( grpc )

    # Adding the following targets:
@ -27,13 +33,12 @@ if ( NOT grpc_POPULATED )
    # grpc++
    add_subdirectory( ${grpc_SOURCE_DIR}
                      ${grpc_BINARY_DIR}
-                      EXCLUDE_FROM_ALL
-                      )
+                      EXCLUDE_FROM_ALL )
 endif()

 # get grpc COMPILE_OPTIONS )
-get_property(var DIRECTORY "${grpc_SOURCE_DIR}" PROPERTY COMPILE_OPTIONS )
-message(STATUS "grpc src compile options: ${var}")
+get_property( var DIRECTORY "${grpc_SOURCE_DIR}" PROPERTY COMPILE_OPTIONS )
+message( STATUS "grpc src compile options: ${var}" )


 # **************************************************************************
@ -53,36 +58,39 @@ get_filename_component( status_proto_path    "${PROTO_PATH}"   PATH )


 # Generated sources
-set(milvus_proto_srcs "${PROTO_PATH}/gen-milvus-test/helloworld.pb.cc")
-set(milvus_proto_hdrs "${PROTO_PATH}/gen-milvus-test/helloworld.pb.h")
-set(milvus_grpc_srcs "${PROTO_PATH}/gen-milvus-test/helloworld.grpc.pb.cc")
-set(milvus_grpc_hdrs "${PROTO_PATH}/gen-milvus-test/helloworld.grpc.pb.h")
+set( milvus_proto_srcs  "${PROTO_PATH}/gen-milvus-test/helloworld.pb.cc" )
+set( milvus_proto_hdrs  "${PROTO_PATH}/gen-milvus-test/helloworld.pb.h" )
+set( milvus_grpc_srcs   "${PROTO_PATH}/gen-milvus-test/helloworld.grpc.pb.cc" )
+set( milvus_grpc_hdrs   "${PROTO_PATH}/gen-milvus-test/helloworld.grpc.pb.h" )

-set(status_proto_srcs "${PROTO_PATH}/gen-status-test/helloworld.pb.cc")
-set(status_proto_hdrs "${PROTO_PATH}/gen-status-test/helloworld.pb.h")
-set(status_grpc_srcs "${PROTO_PATH}/gen-status-test/helloworld.grpc.pb.cc")
-set(status_grpc_hdrs "${PROTO_PATH}/gen-status-test/helloworld.grpc.pb.h")
+set( status_proto_srcs  "${PROTO_PATH}/gen-status-test/helloworld.pb.cc" )
+set( status_proto_hdrs  "${PROTO_PATH}/gen-status-test/helloworld.pb.h" )
+set( status_grpc_srcs   "${PROTO_PATH}/gen-status-test/helloworld.grpc.pb.cc" )
+set( status_grpc_hdrs   "${PROTO_PATH}/gen-status-test/helloworld.grpc.pb.h" )

 add_custom_command(
-      OUTPUT "${milvus_proto_srcs}" "${milvus_proto_hdrs}" "${milvus_grpc_srcs}" "${milvus_grpc_hdrs}"
-      COMMAND ${PROTOC_EXCUTABLE}
-      ARGS  --grpc_out  "${PROTO_PATH}/gen-milvus-test"
+    OUTPUT  "${milvus_proto_srcs}"
+            "${milvus_proto_hdrs}"
+            "${milvus_grpc_srcs}"
+            "${milvus_grpc_hdrs}"
+    COMMAND ${PROTOC_EXCUTABLE}
+    ARGS    --grpc_out  "${PROTO_PATH}/gen-milvus-test"
            --cpp_out   "${PROTO_PATH}/gen-milvus-test"
            -I          "${milvus_proto_path}"
            --plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN_EXCUTABLE}"
            "${milvus_proto}"
-      DEPENDS "${milvus_proto}" )
+    DEPENDS "${milvus_proto}" )

 add_custom_command(
-      OUTPUT "${status_proto_srcs}" "${status_proto_hdrs}" "${status_grpc_srcs}" "${status_grpc_hdrs}"
-      COMMAND ${PROTOC_EXCUTABLE}
-      ARGS  --grpc_out  "${PROTO_PATH}/gen-status-test"
+    OUTPUT  "${status_proto_srcs}" "${status_proto_hdrs}" "${status_grpc_srcs}" "${status_grpc_hdrs}"
+    COMMAND ${PROTOC_EXCUTABLE}
+    ARGS    --grpc_out  "${PROTO_PATH}/gen-status-test"
            --cpp_out   "${PROTO_PATH}/gen-status-test"
            -I          "${status_proto_path}"
            --plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN_EXCUTABLE}"
            "${status_proto}"
-      DEPENDS "${status_proto}" )
+    DEPENDS "${status_proto}" )

 # Include generated *.pb.h files
-include_directories("${MILVUS_SOURCE_DIR}/gen-milvus-test")
-include_directories("${MILVUS_SOURCE_DIR}/gen-status-test")
+include_directories( "${MILVUS_SOURCE_DIR}/gen-milvus-test")
+include_directories( "${MILVUS_SOURCE_DIR}/gen-status-test" )
--- a/core/thirdparty/gtest/CMakeLists.txt
+++ b/core/thirdparty/gtest/CMakeLists.txt
@ -55,5 +55,5 @@ endif()
 # endif()


-get_property(var DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/googletest-src" PROPERTY COMPILE_OPTIONS )
-message(STATUS "gtest compile options: ${var}")
+get_property( var DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/googletest-src" PROPERTY COMPILE_OPTIONS )
+message( STATUS "gtest compile options: ${var}" )
--- a/core/thirdparty/versions.txt
+++ b/core/thirdparty/versions.txt
@ -3,7 +3,7 @@ GTEST_VERSION=1.8.1
 MYSQLPP_VERSION=3.2.4
 PROMETHEUS_VERSION=0.7.0
 SQLITE_VERSION=3280000
-YAMLCPP_VERSION=0.6.2
+YAMLCPP_VERSION=0.6.3
 LIBUNWIND_VERSION=1.3.1
 GPERFTOOLS_VERSION=2.7
 GRPC_VERSION=4b1c02a7edd03a2fec7dec48f37186306c456378
--- a/core/thirdparty/yaml-cpp/CMakeLists.txt
+++ b/core/thirdparty/yaml-cpp/CMakeLists.txt
@ -0,0 +1,50 @@
+#-------------------------------------------------------------------------------
+# Copyright (C) 2019-2020 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+#-------------------------------------------------------------------------------
+
+if ( DEFINED ENV{MILVUS_YAMLCPP_URL} )
+    set( YAMLCPP_SOURCE_URL "$ENV{MILVUS_YAMLCPP_URL}" )
+else()
+    set( YAMLCPP_SOURCE_URL
+        "https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz"
+        "https://gitee.com/quicksilver/yaml-cpp/repository/archive/yaml-cpp-${YAMLCPP_VERSION}.zip" )
+endif()
+
+message( STATUS "Building yaml-cpp-${YAMLCPP_VERSION} from source" )
+FetchContent_Declare(
+    yaml-cpp
+    URL         ${YAMLCPP_SOURCE_URL}
+    URL_MD5     "b45bf1089a382e81f6b661062c10d0c2"
+    SOURCE_DIR  ${CMAKE_CURRENT_BINARY_DIR}/yaml-src
+    BINARY_DIR  ${CMAKE_CURRENT_BINARY_DIR}/yaml-build
+    )
+
+set( YAML_CPP_BUILD_TESTS CACHE BOOL OFF FORCE )
+set( YAML_CPP_BUILD_TOOLS CACHE BOOL OFF FORCE )
+FetchContent_GetProperties( yaml-cpp )
+if ( NOT yaml-cpp_POPULATED )
+
+    FetchContent_Populate( yaml-cpp )
+
+    # Adding the following targets:
+    # yaml-cpp::yaml-cpp, yaml-cpp
+    add_subdirectory( ${yaml-cpp_SOURCE_DIR}
+                      ${yaml-cpp_BINARY_DIR}
+                      EXCLUDE_FROM_ALL )
+
+endif()
+
+get_target_property( YAML_CPP_INCLUDE_DIR yaml-cpp INTERFACE_INCLUDE_DIRECTORIES )
+message( STATUS ${YAML_CPP_INCLUDE_DIR} )
+
+get_property( var DIRECTORY "${yaml-cpp_SOURCE_DIR}" PROPERTY COMPILE_OPTIONS )
+message( STATUS "yaml compile options: ${var}" )