From 144523b48004aefba2ab45fecbe4e6d80514577b Mon Sep 17 00:00:00 2001 From: XuanYang-cn <51370125+XuanYang-cn@users.noreply.github.com> Date: Thu, 20 Aug 2020 19:13:16 +0800 Subject: [PATCH] Optimize download & compile of Prometheus (#3357) * runable prometheus Signed-off-by: yangxuan * add copyright Signed-off-by: yangxuan Co-authored-by: yangxuan --- core/cmake/ThirdPartyPackages.cmake | 99 +---------------------- core/src/CMakeLists.txt | 21 +---- core/src/db/CMakeLists.txt | 10 --- core/src/metrics/CMakeLists.txt | 8 +- core/thirdparty/CMakeLists.txt | 5 ++ core/thirdparty/prometheus/CMakeLists.txt | 49 +++++++++++ 6 files changed, 61 insertions(+), 131 deletions(-) create mode 100644 core/thirdparty/prometheus/CMakeLists.txt diff --git a/core/cmake/ThirdPartyPackages.cmake b/core/cmake/ThirdPartyPackages.cmake index e79d306bc3..0b091ebb1c 100644 --- a/core/cmake/ThirdPartyPackages.cmake +++ b/core/cmake/ThirdPartyPackages.cmake @@ -11,7 +11,6 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES - Prometheus SQLite fiu AWS @@ -27,9 +26,7 @@ foreach (DEPENDENCY ${MILVUS_THIRDPARTY_DEPENDENCIES}) endforeach () macro(build_dependency DEPENDENCY_NAME) - if ("${DEPENDENCY_NAME}" STREQUAL "Prometheus") - build_prometheus() - elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite") + if ("${DEPENDENCY_NAME}" STREQUAL "SQLite") build_sqlite() elseif ("${DEPENDENCY_NAME}" STREQUAL "fiu") build_fiu() @@ -207,13 +204,6 @@ foreach (_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) endforeach () -if (DEFINED ENV{MILVUS_PROMETHEUS_URL}) - set(PROMETHEUS_SOURCE_URL "$ENV{PROMETHEUS_OPENBLAS_URL}") -else () - set(PROMETHEUS_SOURCE_URL - "https://github.com/milvus-io/prometheus-cpp/archive/${PROMETHEUS_VERSION}.zip") -endif () - if (DEFINED ENV{MILVUS_SQLITE_URL}) set(SQLITE_SOURCE_URL "$ENV{MILVUS_SQLITE_URL}") else () @@ -241,93 +231,6 @@ else () set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz") endif () -# ---------------------------------------------------------------------- -# Prometheus - -macro(build_prometheus) - message(STATUS "Building Prometheus-${PROMETHEUS_VERSION} from source") - set(PROMETHEUS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/prometheus_ep-prefix/src/prometheus_ep") - set(PROMETHEUS_STATIC_LIB_NAME prometheus-cpp) - set(PROMETHEUS_CORE_STATIC_LIB - "${PROMETHEUS_PREFIX}/core/${CMAKE_STATIC_LIBRARY_PREFIX}${PROMETHEUS_STATIC_LIB_NAME}-core${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(PROMETHEUS_PUSH_STATIC_LIB - "${PROMETHEUS_PREFIX}/push/${CMAKE_STATIC_LIBRARY_PREFIX}${PROMETHEUS_STATIC_LIB_NAME}-push${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(PROMETHEUS_PULL_STATIC_LIB - "${PROMETHEUS_PREFIX}/pull/${CMAKE_STATIC_LIBRARY_PREFIX}${PROMETHEUS_STATIC_LIB_NAME}-pull${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - - set(PROMETHEUS_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_LIBDIR=lib - -DBUILD_SHARED_LIBS=OFF - "-DCMAKE_INSTALL_PREFIX=${PROMETHEUS_PREFIX}" - -DCMAKE_BUILD_TYPE=Release) - - ExternalProject_Add(prometheus_ep - URL - ${PROMETHEUS_SOURCE_URL} - ${EP_LOG_OPTIONS} - URL_MD5 - "6550819ae4d61c480a55a69f08159413" - CMAKE_ARGS - ${PROMETHEUS_CMAKE_ARGS} - UPDATE_COMMAND - "" - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} - "DESTDIR=${PROMETHEUS_PREFIX}" - install - BUILD_BYPRODUCTS - "${PROMETHEUS_CORE_STATIC_LIB}" - "${PROMETHEUS_PUSH_STATIC_LIB}" - "${PROMETHEUS_PULL_STATIC_LIB}") - - file(MAKE_DIRECTORY "${PROMETHEUS_PREFIX}/push/include") - add_library(prometheus-cpp-push STATIC IMPORTED) - set_target_properties(prometheus-cpp-push - PROPERTIES IMPORTED_LOCATION "${PROMETHEUS_PUSH_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${PROMETHEUS_PREFIX}/push/include") - add_dependencies(prometheus-cpp-push prometheus_ep) - - file(MAKE_DIRECTORY "${PROMETHEUS_PREFIX}/pull/include") - add_library(prometheus-cpp-pull STATIC IMPORTED) - set_target_properties(prometheus-cpp-pull - PROPERTIES IMPORTED_LOCATION "${PROMETHEUS_PULL_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${PROMETHEUS_PREFIX}/pull/include") - add_dependencies(prometheus-cpp-pull prometheus_ep) - - file(MAKE_DIRECTORY "${PROMETHEUS_PREFIX}/core/include") - add_library(prometheus-cpp-core STATIC IMPORTED) - set_target_properties(prometheus-cpp-core - PROPERTIES IMPORTED_LOCATION "${PROMETHEUS_CORE_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${PROMETHEUS_PREFIX}/core/include") - add_dependencies(prometheus-cpp-core prometheus_ep) -endmacro() - -if (MILVUS_WITH_PROMETHEUS) - - resolve_dependency(Prometheus) - - link_directories(SYSTEM ${PROMETHEUS_PREFIX}/push/) - include_directories(SYSTEM ${PROMETHEUS_PREFIX}/push/include) - - link_directories(SYSTEM ${PROMETHEUS_PREFIX}/pull/) - include_directories(SYSTEM ${PROMETHEUS_PREFIX}/pull/include) - - link_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/) - include_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/include) - -endif () # ---------------------------------------------------------------------- # SQLite diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index c34394a653..8b1950f2f7 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -65,28 +65,11 @@ set(link_lib oatpp query utils - log - ) + log ) -if (MILVUS_WITH_PROMETHEUS) - set(link_lib - ${link_lib} - # dependency prometheus - prometheus-cpp-push - prometheus-cpp-pull - prometheus-cpp-core - ) -endif () -set(link_lib - ${link_lib} - curl - ) - -target_link_libraries( server - PUBLIC ${link_lib} - ) +target_link_libraries( server PUBLIC ${link_lib} ) # **************************** Get&Print Include Directories **************************** get_property( dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES ) diff --git a/core/src/db/CMakeLists.txt b/core/src/db/CMakeLists.txt index 7cf79cfd0e..c4ea17561f 100644 --- a/core/src/db/CMakeLists.txt +++ b/core/src/db/CMakeLists.txt @@ -80,12 +80,6 @@ set( BOOST_LIB libboost_system.a libboost_serialization.a ) -set( PROMETHEUS_LIB prometheus-cpp-push - prometheus-cpp-pull - prometheus-cpp-core - curl - ) - set( THIRD_PARTY_LIBS sqlite yaml-cpp # log mysqlpp @@ -114,10 +108,6 @@ if ( ENABLE_CPU_PROFILING ) PUBLIC gperftools ) endif () -if ( MILVUS_WITH_PROMETHEUS ) - list( APPEND THIRD_PARTY_LIBS ${PROMETHEUS_LIB} ) -endif () - if ( MILVS_WITH_AWS ) list(APPEND THIRD_PARTY_LIBS PUBLIC ${S3_CLIENT_LIB} diff --git a/core/src/metrics/CMakeLists.txt b/core/src/metrics/CMakeLists.txt index 04739c0c85..718a28c8ca 100644 --- a/core/src/metrics/CMakeLists.txt +++ b/core/src/metrics/CMakeLists.txt @@ -16,9 +16,9 @@ aux_source_directory( ${MILVUS_ENGINE_SRC}/metrics/prometheus METRICS_PROMETHE set( METRICS_LIB yaml-cpp ) -set( PROMETHEUS_LIB prometheus-cpp-push - prometheus-cpp-pull - prometheus-cpp-core +set( PROMETHEUS_LIB prometheus-cpp::core + prometheus-cpp::pull + prometheus-cpp::push ) add_library( metrics STATIC ${METRICS_FILES}) @@ -26,7 +26,7 @@ add_library( metrics STATIC ${METRICS_FILES}) if ( MILVUS_WITH_PROMETHEUS ) target_sources( metrics PRIVATE ${METRICS_PROMETHEUS_FILES} ) add_dependencies( metrics ${PROMETHEUS_LIB} ) - list( APPEND metrics_lib ${PROMETHEUS_LIB} ) + list( APPEND METRICS_LIB ${PROMETHEUS_LIB} ) endif () diff --git a/core/thirdparty/CMakeLists.txt b/core/thirdparty/CMakeLists.txt index f550e763f1..8e8cd85c17 100644 --- a/core/thirdparty/CMakeLists.txt +++ b/core/thirdparty/CMakeLists.txt @@ -71,3 +71,8 @@ endif () if ( MILVUS_WITH_MYSQLPP ) add_subdirectory( mysqlpp ) endif () + +# ****************************** Thirdparty prometheus *************************************** +if (MILVUS_WITH_PROMETHEUS) + add_subdirectory( prometheus ) +endif () diff --git a/core/thirdparty/prometheus/CMakeLists.txt b/core/thirdparty/prometheus/CMakeLists.txt new file mode 100644 index 0000000000..1c20fd852a --- /dev/null +++ b/core/thirdparty/prometheus/CMakeLists.txt @@ -0,0 +1,49 @@ +#------------------------------------------------------------------------------- +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. +#------------------------------------------------------------------------------- +if ( DEFINED ENV{MILVUS_PROMETHEUS_URL} ) + set( PROMETHEUS_SOURCE_URL "$ENV{PROMETHEUS_OPENBLAS_URL}" ) +else () + set( PROMETHEUS_SOURCE_URL + "https://github.com/milvus-io/prometheus-cpp/archive/${PROMETHEUS_VERSION}.zip" ) +endif () + +# ---------------------------------------------------------------------- +# Prometheus + +message( STATUS "Building Prometheus-${PROMETHEUS_VERSION} from source" ) + +FetchContent_Declare( + prometheus + URL ${PROMETHEUS_SOURCE_URL} + URL_MD5 "6550819ae4d61c480a55a69f08159413" + SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/prometheus-src + BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/prometheus-build + DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} ) + +set( ENABLE_TESTING OFF CACHE BOOL "" FORCE ) +FetchContent_GetProperties( prometheus ) +if ( NOT prometheus_POPULATED ) + FetchContent_Populate( prometheus ) + + # Adding the following targets: + # prometheus-cpp::core + # prometheus-cpp::pull + # prometheus-cpp::push + add_subdirectory( ${prometheus_SOURCE_DIR} + ${prometheus_BINARY_DIR} + EXCLUDE_FROM_ALL ) +endif() + +# get prometheus COMPILE_OPTIONS +get_property( var DIRECTORY "${prometheus_SOURCE_DIR}" PROPERTY COMPILE_OPTIONS ) +message( STATUS "prometheus src compile options: ${var}" )