diff --git a/CHANGELOG.md b/CHANGELOG.md index 327b1f9b3e..25659aed15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#260 - C++ SDK README - \#314 - add Find FAISS in CMake - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue +- \#322 - Add option to enable / disable prometheus ## Task diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 45a2bc5fe6..f24935f788 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -33,7 +33,7 @@ message(STATUS "Build time = ${BUILD_TIME}") MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'" - OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) ENDMACRO(GET_GIT_BRANCH_NAME) GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME) @@ -117,17 +117,17 @@ include(DefineOptions) include(BuildUtils) include(ThirdPartyPackages) -if(MILVUS_USE_CCACHE) - find_program(CCACHE_FOUND ccache) - if(CCACHE_FOUND) - message(STATUS "Using ccache: ${CCACHE_FOUND}") - set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND}) - set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) - # let ccache preserve C++ comments, because some of them may be - # meaningful to the compiler - set(ENV{CCACHE_COMMENTS} "1") - endif(CCACHE_FOUND) -endif() +if (MILVUS_USE_CCACHE) + find_program(CCACHE_FOUND ccache) + if (CCACHE_FOUND) + message(STATUS "Using ccache: ${CCACHE_FOUND}") + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND}) + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) + # let ccache preserve C++ comments, because some of them may be + # meaningful to the compiler + set(ENV{CCACHE_COMMENTS} "1") + endif (CCACHE_FOUND) +endif () set(MILVUS_CPU_VERSION false) if (MILVUS_GPU_VERSION) @@ -142,6 +142,10 @@ else () add_compile_definitions("MILVUS_CPU_VERSION") endif () +if (MILVUS_WITH_PROMETHEUS) + add_compile_definitions("MILVUS_WITH_PROMETHEUS") +endif () + if (CMAKE_BUILD_TYPE STREQUAL "Release") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") if (MILVUS_GPU_VERSION) @@ -176,9 +180,9 @@ endif () if (MILVUS_GPU_VERSION) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_gpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) -else() +else () configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_cpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) -endif() +endif () configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf) diff --git a/core/build.sh b/core/build.sh index 9b690a0261..85dea0ab88 100755 --- a/core/build.sh +++ b/core/build.sh @@ -14,10 +14,11 @@ CUSTOMIZATION="OFF" # default use ori faiss CUDA_COMPILER=/usr/local/cuda/bin/nvcc GPU_VERSION="OFF" #defaults to CPU version WITH_MKL="OFF" -FAISS_ROOT="" +FAISS_ROOT="" #FAISS root path FAISS_SOURCE="BUNDLED" +WITH_PROMETHEUS="ON" -while getopts "p:d:t:f:ulrcgjhxzm" arg +while getopts "p:d:t:f:ulrcgjhxzme" arg do case $arg in p) @@ -63,7 +64,10 @@ do ;; m) WITH_MKL="ON" - ;; + ;; + e) + WITH_PROMETHEUS="OFF" + ;; h) # help echo " @@ -80,10 +84,11 @@ parameter: -j: use jfrog cache build directory(default: OFF) -g: build GPU version(default: OFF) -m: build with MKL(default: OFF) +-e: build without prometheus -h: help usage: -./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h] +./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-e] [-h] " exit 0 ;; @@ -118,6 +123,7 @@ CMAKE_CMD="cmake \ -DCUSTOMIZATION=${CUSTOMIZATION} \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ -DFAISS_WITH_MKL=${WITH_MKL} \ +-DMILVUS_WITH_PROMETHEUS=${WITH_PROMETHEUS} \ ../" echo ${CMAKE_CMD} ${CMAKE_CMD} diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 937c0716a4..aa79c084f1 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -37,6 +37,7 @@ endforeach () aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/metrics/prometheus metrics_prometheus_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) @@ -91,6 +92,11 @@ set(engine_files ${wrapper_files} ) +if (MILVUS_WITH_PROMETHEUS) + set(engine_files ${engine_files} + ${metrics_prometheus_files}) +endif () + set(client_grpc_lib grpcpp_channelz grpc++ @@ -115,7 +121,6 @@ set(third_party_libs sqlite ${client_grpc_lib} yaml-cpp - ${prometheus_lib} mysqlpp zlib ${boost_lib} @@ -138,13 +143,19 @@ if (MILVUS_GPU_VERSION) ) endif () -if (MILVUS_ENABLE_PROFILING STREQUAL "ON") +if (MILVUS_ENABLE_PROFILING) set(third_party_libs ${third_party_libs} gperftools libunwind ) endif () +if (MILVUS_WITH_PROMETHEUS) + set(third_party_libs ${third_party_libs} + ${prometheus_lib} + ) +endif () + set(engine_libs pthread libgomp.a @@ -166,13 +177,22 @@ target_link_libraries(milvus_engine ${engine_libs} ) -add_library(metrics STATIC ${metrics_files}) +if (MILVUS_WITH_PROMETHEUS) + add_library(metrics STATIC ${metrics_files} ${metrics_prometheus_files}) +else () + add_library(metrics STATIC ${metrics_files}) +endif () set(metrics_lib yaml-cpp - ${prometheus_lib} ) +if (MILVUS_WITH_PROMETHEUS) + set(metrics_lib ${metrics_lib} + ${prometheus_lib} + ) +endif () + target_link_libraries(metrics ${metrics_lib}) set(server_libs diff --git a/core/src/metrics/Metrics.cpp b/core/src/metrics/Metrics.cpp index 51db5555b8..5fd3553cdc 100644 --- a/core/src/metrics/Metrics.cpp +++ b/core/src/metrics/Metrics.cpp @@ -16,8 +16,10 @@ // under the License. #include "metrics/Metrics.h" -#include "PrometheusMetrics.h" #include "server/Config.h" +#ifdef MILVUS_WITH_PROMETHEUS +#include "metrics/prometheus/PrometheusMetrics.h" +#endif #include @@ -37,11 +39,15 @@ Metrics::CreateMetricsCollector() { config.GetMetricConfigCollector(collector_type_str); +#ifdef MILVUS_WITH_PROMETHEUS if (collector_type_str == "prometheus") { return PrometheusMetrics::GetInstance(); } else { return MetricsBase::GetInstance(); } +#else + return MetricsBase::GetInstance(); +#endif } } // namespace server diff --git a/core/src/metrics/PrometheusMetrics.cpp b/core/src/metrics/prometheus/PrometheusMetrics.cpp similarity index 98% rename from core/src/metrics/PrometheusMetrics.cpp rename to core/src/metrics/prometheus/PrometheusMetrics.cpp index 770b34dc47..19b2683280 100644 --- a/core/src/metrics/PrometheusMetrics.cpp +++ b/core/src/metrics/prometheus/PrometheusMetrics.cpp @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include "metrics/PrometheusMetrics.h" -#include "SystemInfo.h" +#include "metrics/prometheus/PrometheusMetrics.h" #include "cache/GpuCacheMgr.h" +#include "metrics/SystemInfo.h" #include "server/Config.h" #include "utils/Log.h" diff --git a/core/src/metrics/PrometheusMetrics.h b/core/src/metrics/prometheus/PrometheusMetrics.h similarity index 99% rename from core/src/metrics/PrometheusMetrics.h rename to core/src/metrics/prometheus/PrometheusMetrics.h index ef60f9a231..5a452ca02c 100644 --- a/core/src/metrics/PrometheusMetrics.h +++ b/core/src/metrics/prometheus/PrometheusMetrics.h @@ -24,7 +24,7 @@ #include #include -#include "MetricBase.h" +#include "metrics/MetricBase.h" #include "utils/Error.h" #define METRICS_NOW_TIME std::chrono::system_clock::now() diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 10ab362e77..01e1054f7e 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -110,12 +110,18 @@ set(unittest_libs pthread metrics gfortran - prometheus-cpp-pull - prometheus-cpp-push - prometheus-cpp-core - dl - z ) +if (MILVUS_WITH_PROMETHEUS) + set(unittest_libs ${unittest_libs} + prometheus-cpp-push + prometheus-cpp-pull + prometheus-cpp-core + ) +endif () +set(unittest_libs ${unittest_libs} + dl + z + ) if (MILVUS_GPU_VERSION) include_directories("${CUDA_INCLUDE_DIRS}") @@ -135,4 +141,4 @@ add_subdirectory(db) add_subdirectory(wrapper) add_subdirectory(metrics) add_subdirectory(scheduler) -add_subdirectory(server) \ No newline at end of file +add_subdirectory(server) diff --git a/core/unittest/metrics/CMakeLists.txt b/core/unittest/metrics/CMakeLists.txt index ad9d4e3943..11c25a71e4 100644 --- a/core/unittest/metrics/CMakeLists.txt +++ b/core/unittest/metrics/CMakeLists.txt @@ -18,10 +18,15 @@ #------------------------------------------------------------------------------- set(test_files - ${CMAKE_CURRENT_SOURCE_DIR}/test_metricbase.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/test_metrics.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/test_prometheus.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/utils.cpp) + test_metricbase.cpp + test_metrics.cpp + utils.cpp + ) + +if (MILVUS_WITH_PROMETHEUS) + set(test_files ${test_files} + test_prometheus.cpp) +endif () add_executable(test_metrics ${common_files} diff --git a/core/unittest/metrics/test_prometheus.cpp b/core/unittest/metrics/test_prometheus.cpp index 50e845d1c2..6e339b73b4 100644 --- a/core/unittest/metrics/test_prometheus.cpp +++ b/core/unittest/metrics/test_prometheus.cpp @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include "metrics/PrometheusMetrics.h" #include "server/Config.h" +#include "metrics/prometheus/PrometheusMetrics.h" #include #include