From c029c4007d89bb26205e0d3c841af3d746309acb Mon Sep 17 00:00:00 2001 From: zhenwu Date: Fri, 8 Nov 2019 14:09:03 +0800 Subject: [PATCH 01/16] add cpu deploy docker image --- docker/deploy/cpu_version/Dockerfile | 18 ++++++++++++++++++ .../docker-entrypoint.sh | 0 .../{ => gpu_version}/ubuntu16.04/Dockerfile | 0 .../ubuntu16.04}/docker-entrypoint.sh | 0 .../{ => gpu_version}/ubuntu18.04/Dockerfile | 0 .../ubuntu18.04/docker-entrypoint.sh | 10 ++++++++++ 6 files changed, 28 insertions(+) create mode 100644 docker/deploy/cpu_version/Dockerfile rename docker/deploy/{ubuntu16.04 => cpu_version}/docker-entrypoint.sh (100%) rename docker/deploy/{ => gpu_version}/ubuntu16.04/Dockerfile (100%) rename docker/deploy/{ubuntu18.04 => gpu_version/ubuntu16.04}/docker-entrypoint.sh (100%) rename docker/deploy/{ => gpu_version}/ubuntu18.04/Dockerfile (100%) create mode 100755 docker/deploy/gpu_version/ubuntu18.04/docker-entrypoint.sh diff --git a/docker/deploy/cpu_version/Dockerfile b/docker/deploy/cpu_version/Dockerfile new file mode 100644 index 0000000000..317001385e --- /dev/null +++ b/docker/deploy/cpu_version/Dockerfile @@ -0,0 +1,18 @@ +FROM ubuntu:18.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gfortran libsqlite3-dev libmysqlclient-dev && \ + apt-get remove --purge -y && \ + rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so + +COPY ./docker-entrypoint.sh /opt +COPY ./milvus /opt/milvus +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/milvus/lib" + +ENTRYPOINT [ "/opt/docker-entrypoint.sh" ] + +CMD [ "start" ] + +EXPOSE 19530 diff --git a/docker/deploy/ubuntu16.04/docker-entrypoint.sh b/docker/deploy/cpu_version/docker-entrypoint.sh similarity index 100% rename from docker/deploy/ubuntu16.04/docker-entrypoint.sh rename to docker/deploy/cpu_version/docker-entrypoint.sh diff --git a/docker/deploy/ubuntu16.04/Dockerfile b/docker/deploy/gpu_version/ubuntu16.04/Dockerfile similarity index 100% rename from docker/deploy/ubuntu16.04/Dockerfile rename to docker/deploy/gpu_version/ubuntu16.04/Dockerfile diff --git a/docker/deploy/ubuntu18.04/docker-entrypoint.sh b/docker/deploy/gpu_version/ubuntu16.04/docker-entrypoint.sh similarity index 100% rename from docker/deploy/ubuntu18.04/docker-entrypoint.sh rename to docker/deploy/gpu_version/ubuntu16.04/docker-entrypoint.sh diff --git a/docker/deploy/ubuntu18.04/Dockerfile b/docker/deploy/gpu_version/ubuntu18.04/Dockerfile similarity index 100% rename from docker/deploy/ubuntu18.04/Dockerfile rename to docker/deploy/gpu_version/ubuntu18.04/Dockerfile diff --git a/docker/deploy/gpu_version/ubuntu18.04/docker-entrypoint.sh b/docker/deploy/gpu_version/ubuntu18.04/docker-entrypoint.sh new file mode 100755 index 0000000000..446c174d74 --- /dev/null +++ b/docker/deploy/gpu_version/ubuntu18.04/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +if [ "$1" == 'start' ]; then + cd /opt/milvus/scripts && ./start_server.sh +fi + +exec "$@" + From bf0e5cbc9da07ae8fafba45bcf73a53e18c3ced6 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Thu, 14 Nov 2019 11:22:07 +0800 Subject: [PATCH 02/16] Add option to disable/enble prometheus when compiling --- core/CMakeLists.txt | 32 +++++++++++-------- core/src/CMakeLists.txt | 28 +++++++++++++--- core/src/metrics/Metrics.cpp | 8 ++++- .../{ => prometheus}/PrometheusMetrics.cpp | 4 +-- .../{ => prometheus}/PrometheusMetrics.h | 2 +- core/unittest/CMakeLists.txt | 10 ++++-- core/unittest/metrics/CMakeLists.txt | 11 ++++++- core/unittest/metrics/test_prometheus.cpp | 2 +- 8 files changed, 70 insertions(+), 27 deletions(-) rename core/src/metrics/{ => prometheus}/PrometheusMetrics.cpp (99%) rename core/src/metrics/{ => prometheus}/PrometheusMetrics.h (99%) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 2e6fdfc46e..6f4f3a3703 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -33,7 +33,7 @@ message(STATUS "Build time = ${BUILD_TIME}") MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'" - OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) ENDMACRO(GET_GIT_BRANCH_NAME) GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME) @@ -117,17 +117,17 @@ include(DefineOptions) include(BuildUtils) include(ThirdPartyPackages) -if(MILVUS_USE_CCACHE) - find_program(CCACHE_FOUND ccache) - if(CCACHE_FOUND) - message(STATUS "Using ccache: ${CCACHE_FOUND}") - set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND}) - set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) - # let ccache preserve C++ comments, because some of them may be - # meaningful to the compiler - set(ENV{CCACHE_COMMENTS} "1") - endif(CCACHE_FOUND) -endif() +if (MILVUS_USE_CCACHE) + find_program(CCACHE_FOUND ccache) + if (CCACHE_FOUND) + message(STATUS "Using ccache: ${CCACHE_FOUND}") + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND}) + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) + # let ccache preserve C++ comments, because some of them may be + # meaningful to the compiler + set(ENV{CCACHE_COMMENTS} "1") + endif (CCACHE_FOUND) +endif () set(MILVUS_CPU_VERSION false) if (MILVUS_GPU_VERSION) @@ -142,6 +142,10 @@ else () add_compile_definitions("MILVUS_CPU_VERSION") endif () +if (MILVUS_WITH_PROMETHEUS) + add_compile_definitions("MILVUS_WITH_PROMETHEUS") +endif () + if (CMAKE_BUILD_TYPE STREQUAL "Release") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") if (MILVUS_GPU_VERSION) @@ -176,9 +180,9 @@ endif () if (MILVUS_GPU_VERSION) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_gpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) -else() +else () configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_cpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) -endif() +endif () configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf) diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 937c0716a4..aa79c084f1 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -37,6 +37,7 @@ endforeach () aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/metrics/prometheus metrics_prometheus_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) @@ -91,6 +92,11 @@ set(engine_files ${wrapper_files} ) +if (MILVUS_WITH_PROMETHEUS) + set(engine_files ${engine_files} + ${metrics_prometheus_files}) +endif () + set(client_grpc_lib grpcpp_channelz grpc++ @@ -115,7 +121,6 @@ set(third_party_libs sqlite ${client_grpc_lib} yaml-cpp - ${prometheus_lib} mysqlpp zlib ${boost_lib} @@ -138,13 +143,19 @@ if (MILVUS_GPU_VERSION) ) endif () -if (MILVUS_ENABLE_PROFILING STREQUAL "ON") +if (MILVUS_ENABLE_PROFILING) set(third_party_libs ${third_party_libs} gperftools libunwind ) endif () +if (MILVUS_WITH_PROMETHEUS) + set(third_party_libs ${third_party_libs} + ${prometheus_lib} + ) +endif () + set(engine_libs pthread libgomp.a @@ -166,13 +177,22 @@ target_link_libraries(milvus_engine ${engine_libs} ) -add_library(metrics STATIC ${metrics_files}) +if (MILVUS_WITH_PROMETHEUS) + add_library(metrics STATIC ${metrics_files} ${metrics_prometheus_files}) +else () + add_library(metrics STATIC ${metrics_files}) +endif () set(metrics_lib yaml-cpp - ${prometheus_lib} ) +if (MILVUS_WITH_PROMETHEUS) + set(metrics_lib ${metrics_lib} + ${prometheus_lib} + ) +endif () + target_link_libraries(metrics ${metrics_lib}) set(server_libs diff --git a/core/src/metrics/Metrics.cpp b/core/src/metrics/Metrics.cpp index 51db5555b8..5fd3553cdc 100644 --- a/core/src/metrics/Metrics.cpp +++ b/core/src/metrics/Metrics.cpp @@ -16,8 +16,10 @@ // under the License. #include "metrics/Metrics.h" -#include "PrometheusMetrics.h" #include "server/Config.h" +#ifdef MILVUS_WITH_PROMETHEUS +#include "metrics/prometheus/PrometheusMetrics.h" +#endif #include @@ -37,11 +39,15 @@ Metrics::CreateMetricsCollector() { config.GetMetricConfigCollector(collector_type_str); +#ifdef MILVUS_WITH_PROMETHEUS if (collector_type_str == "prometheus") { return PrometheusMetrics::GetInstance(); } else { return MetricsBase::GetInstance(); } +#else + return MetricsBase::GetInstance(); +#endif } } // namespace server diff --git a/core/src/metrics/PrometheusMetrics.cpp b/core/src/metrics/prometheus/PrometheusMetrics.cpp similarity index 99% rename from core/src/metrics/PrometheusMetrics.cpp rename to core/src/metrics/prometheus/PrometheusMetrics.cpp index 770b34dc47..cb6fd07e19 100644 --- a/core/src/metrics/PrometheusMetrics.cpp +++ b/core/src/metrics/prometheus/PrometheusMetrics.cpp @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -#include "metrics/PrometheusMetrics.h" -#include "SystemInfo.h" +#include "PrometheusMetrics.h" #include "cache/GpuCacheMgr.h" #include "server/Config.h" +#include "metrics/SystemInfo.h" #include "utils/Log.h" #include diff --git a/core/src/metrics/PrometheusMetrics.h b/core/src/metrics/prometheus/PrometheusMetrics.h similarity index 99% rename from core/src/metrics/PrometheusMetrics.h rename to core/src/metrics/prometheus/PrometheusMetrics.h index ef60f9a231..5a452ca02c 100644 --- a/core/src/metrics/PrometheusMetrics.h +++ b/core/src/metrics/prometheus/PrometheusMetrics.h @@ -24,7 +24,7 @@ #include #include -#include "MetricBase.h" +#include "metrics/MetricBase.h" #include "utils/Error.h" #define METRICS_NOW_TIME std::chrono::system_clock::now() diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 10ab362e77..82aaa9efd1 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -110,12 +110,16 @@ set(unittest_libs pthread metrics gfortran - prometheus-cpp-pull - prometheus-cpp-push - prometheus-cpp-core dl z ) +if (MILVUS_WITH_PROMETHEUS) + set(unittest_libs ${unittest_libs} + prometheus-cpp-push + prometheus-cpp-pull + prometheus-cpp-core + ) +endif () if (MILVUS_GPU_VERSION) include_directories("${CUDA_INCLUDE_DIRS}") diff --git a/core/unittest/metrics/CMakeLists.txt b/core/unittest/metrics/CMakeLists.txt index eba8146baa..11c25a71e4 100644 --- a/core/unittest/metrics/CMakeLists.txt +++ b/core/unittest/metrics/CMakeLists.txt @@ -17,7 +17,16 @@ # under the License. #------------------------------------------------------------------------------- -aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files) +set(test_files + test_metricbase.cpp + test_metrics.cpp + utils.cpp + ) + +if (MILVUS_WITH_PROMETHEUS) + set(test_files ${test_files} + test_prometheus.cpp) +endif () add_executable(test_metrics ${common_files} diff --git a/core/unittest/metrics/test_prometheus.cpp b/core/unittest/metrics/test_prometheus.cpp index 50e845d1c2..6e339b73b4 100644 --- a/core/unittest/metrics/test_prometheus.cpp +++ b/core/unittest/metrics/test_prometheus.cpp @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include "metrics/PrometheusMetrics.h" #include "server/Config.h" +#include "metrics/prometheus/PrometheusMetrics.h" #include #include From 981d5393fff43ecfd00f0c34cbd144d97c66c7fd Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Thu, 14 Nov 2019 14:08:19 +0800 Subject: [PATCH 03/16] update build.sh --- core/build.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/core/build.sh b/core/build.sh index 9b690a0261..d8d73cabdb 100755 --- a/core/build.sh +++ b/core/build.sh @@ -14,10 +14,11 @@ CUSTOMIZATION="OFF" # default use ori faiss CUDA_COMPILER=/usr/local/cuda/bin/nvcc GPU_VERSION="OFF" #defaults to CPU version WITH_MKL="OFF" -FAISS_ROOT="" +FAISS_ROOT="" #FAISS root path FAISS_SOURCE="BUNDLED" +WITH_PROMETHEUS="ON" -while getopts "p:d:t:f:ulrcgjhxzm" arg +while getopts "p:d:t:f:ulrcgjhxzme" arg do case $arg in p) @@ -63,7 +64,10 @@ do ;; m) WITH_MKL="ON" - ;; + ;; + e) + WITH_PROMETHEUS="OFF" + ;; h) # help echo " @@ -80,10 +84,11 @@ parameter: -j: use jfrog cache build directory(default: OFF) -g: build GPU version(default: OFF) -m: build with MKL(default: OFF) +-e: build without prometheus(default: OFF) -h: help usage: -./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h] +./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-e] [-h] " exit 0 ;; @@ -118,6 +123,7 @@ CMAKE_CMD="cmake \ -DCUSTOMIZATION=${CUSTOMIZATION} \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ -DFAISS_WITH_MKL=${WITH_MKL} \ +-DMILVUS_WITH_PROMETHEUS=${WITH_PROMETHEUS} \ ../" echo ${CMAKE_CMD} ${CMAKE_CMD} From 5a0e63ed388f7dcfb6e8373bcc1630d470c63668 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Thu, 14 Nov 2019 18:20:46 +0800 Subject: [PATCH 04/16] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 327b1f9b3e..25659aed15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#260 - C++ SDK README - \#314 - add Find FAISS in CMake - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue +- \#322 - Add option to enable / disable prometheus ## Task From bb5f38b0ddff3b0450ccf2baeeedbbd87b28b240 Mon Sep 17 00:00:00 2001 From: groot Date: Thu, 14 Nov 2019 19:25:53 +0800 Subject: [PATCH 05/16] #316 Some files not merged after vectors added --- CHANGELOG.md | 1 + core/src/db/DBImpl.cpp | 18 +++++++++++++++--- core/src/db/meta/MySQLMetaImpl.cpp | 13 ++++++++----- core/src/db/meta/SqliteMetaImpl.cpp | 13 +++++++++---- 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 327b1f9b3e..bb0951ed0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Please mark all change in change log and use the ticket from JIRA. ## Bug - \#246 - Exclude src/external folder from code coverage for jenkin ci - \#248 - Reside src/external in thirdparty +- \#316 - Some files not merged after vectors added ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 3e0501b84e..9898751c9f 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -619,6 +619,18 @@ DBImpl::StartCompactionTask() { { std::lock_guard lck(compact_result_mutex_); if (compact_thread_results_.empty()) { + // collect merge files for all tables(if compact_table_ids_ is empty) for two reasons: + // 1. other tables may still has un-merged files + // 2. server may be closed unexpected, these un-merge files need to be merged when server restart + if(compact_table_ids_.empty()) { + std::vector table_schema_array; + meta_ptr_->AllTables(table_schema_array); + for( auto& schema : table_schema_array) { + compact_table_ids_.insert(schema.table_id_); + } + } + + // start merge file thread compact_thread_results_.push_back( compact_thread_pool_.enqueue(&DBImpl::BackgroundCompaction, this, compact_table_ids_)); compact_table_ids_.clear(); @@ -717,7 +729,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) { for (auto& kv : raw_files) { auto files = kv.second; if (files.size() < options_.merge_trigger_number_) { - ENGINE_LOG_DEBUG << "Files number not greater equal than merge trigger number, skip merge action"; + ENGINE_LOG_TRACE << "Files number not greater equal than merge trigger number, skip merge action"; continue; } @@ -734,7 +746,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) { void DBImpl::BackgroundCompaction(std::set table_ids) { - ENGINE_LOG_TRACE << " Background compaction thread start"; + ENGINE_LOG_TRACE << "Background compaction thread start"; Status status; for (auto& table_id : table_ids) { @@ -757,7 +769,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { } meta_ptr_->CleanUpFilesWithTTL(ttl); - ENGINE_LOG_TRACE << " Background compaction thread exit"; + ENGINE_LOG_TRACE << "Background compaction thread exit"; } void diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index ff36554c10..bf83447806 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1392,6 +1392,7 @@ MySQLMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFiles } // Scoped Connection Status ret; + int64_t to_merge_files = 0; for (auto& resRow : res) { TableFileSchema table_file; table_file.file_size_ = resRow["file_size"]; @@ -1420,13 +1421,14 @@ MySQLMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFiles auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { files[table_file.date_] = TableFilesSchema(); + to_merge_files++; } files[table_file.date_].push_back(table_file); } - if (res.size() > 0) { - ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-merge files"; + if (to_merge_files > 0) { + ENGINE_LOG_TRACE << "Collect " << to_merge_files << " to-merge files"; } return ret; } catch (std::exception& e) { @@ -1809,6 +1811,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + int64_t remove_tables = 0; if (!res.empty()) { std::stringstream idsToDeleteSS; for (auto& resRow : res) { @@ -1817,7 +1820,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { resRow["table_id"].to_string(table_id); utils::DeleteTablePath(options_, table_id, false); // only delete empty folder - + remove_tables++; idsToDeleteSS << "id = " << std::to_string(id) << " OR "; } std::string idsToDeleteStr = idsToDeleteSS.str(); @@ -1832,8 +1835,8 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } } - if (res.size() > 0) { - ENGINE_LOG_DEBUG << "Remove " << res.size() << " tables from meta"; + if (remove_tables > 0) { + ENGINE_LOG_DEBUG << "Remove " << remove_tables << " tables from meta"; } } // Scoped Connection } catch (std::exception& e) { diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 6221dd8ac1..22e953fe9d 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -971,6 +971,7 @@ SqliteMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFile order_by(&TableFileSchema::file_size_).desc()); Status result; + int64_t to_merge_files = 0; for (auto& file : selected) { TableFileSchema table_file; table_file.file_size_ = std::get<4>(file); @@ -999,11 +1000,13 @@ SqliteMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFile if (dateItr == files.end()) { files[table_file.date_] = TableFilesSchema(); } + files[table_file.date_].push_back(table_file); + to_merge_files++; } - if (selected.size() > 0) { - ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-merge files"; + if (to_merge_files > 0) { + ENGINE_LOG_TRACE << "Collect " << to_merge_files << " to-merge files"; } return result; } catch (std::exception& e) { @@ -1313,16 +1316,18 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { try { server::MetricCollector metric; + int64_t remove_tables = 0; for (auto& table_id : table_ids) { auto selected = ConnectorPtr->select(columns(&TableFileSchema::file_id_), where(c(&TableFileSchema::table_id_) == table_id)); if (selected.size() == 0) { utils::DeleteTablePath(options_, table_id); + remove_tables++; } } - if (table_ids.size() > 0) { - ENGINE_LOG_DEBUG << "Remove " << table_ids.size() << " tables folder"; + if (remove_tables) { + ENGINE_LOG_DEBUG << "Remove " << remove_tables << " tables folder"; } } catch (std::exception& e) { return HandleException("Encounter exception when delete table folder", e.what()); From 94c2427f55c3dab2c0bc09c57c95595b8a757687 Mon Sep 17 00:00:00 2001 From: groot Date: Thu, 14 Nov 2019 19:39:35 +0800 Subject: [PATCH 06/16] code lint --- core/src/db/DBImpl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 9898751c9f..d688c88ff2 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -622,10 +622,10 @@ DBImpl::StartCompactionTask() { // collect merge files for all tables(if compact_table_ids_ is empty) for two reasons: // 1. other tables may still has un-merged files // 2. server may be closed unexpected, these un-merge files need to be merged when server restart - if(compact_table_ids_.empty()) { + if (compact_table_ids_.empty()) { std::vector table_schema_array; meta_ptr_->AllTables(table_schema_array); - for( auto& schema : table_schema_array) { + for (auto& schema : table_schema_array) { compact_table_ids_.insert(schema.table_id_); } } From f96bd076ec31db8635430759504fac661236321c Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Thu, 14 Nov 2019 19:57:12 +0800 Subject: [PATCH 07/16] format --- core/src/metrics/prometheus/PrometheusMetrics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/metrics/prometheus/PrometheusMetrics.cpp b/core/src/metrics/prometheus/PrometheusMetrics.cpp index cb6fd07e19..2dc48146b8 100644 --- a/core/src/metrics/prometheus/PrometheusMetrics.cpp +++ b/core/src/metrics/prometheus/PrometheusMetrics.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "PrometheusMetrics.h" +#include "prometheus/PrometheusMetrics.h" #include "cache/GpuCacheMgr.h" #include "server/Config.h" #include "metrics/SystemInfo.h" From e7e416775bedd7f47da86d548adca181db4c2d6b Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Thu, 14 Nov 2019 20:00:32 +0800 Subject: [PATCH 08/16] update build.sh --- core/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/build.sh b/core/build.sh index d8d73cabdb..85dea0ab88 100755 --- a/core/build.sh +++ b/core/build.sh @@ -84,7 +84,7 @@ parameter: -j: use jfrog cache build directory(default: OFF) -g: build GPU version(default: OFF) -m: build with MKL(default: OFF) --e: build without prometheus(default: OFF) +-e: build without prometheus -h: help usage: From 3bcd01fe55435e5b4eb02aa1df6737bb3cbcebaf Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Thu, 14 Nov 2019 20:07:19 +0800 Subject: [PATCH 09/16] format --- core/src/metrics/prometheus/PrometheusMetrics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/metrics/prometheus/PrometheusMetrics.cpp b/core/src/metrics/prometheus/PrometheusMetrics.cpp index 2dc48146b8..e55c01b5d0 100644 --- a/core/src/metrics/prometheus/PrometheusMetrics.cpp +++ b/core/src/metrics/prometheus/PrometheusMetrics.cpp @@ -17,8 +17,8 @@ #include "prometheus/PrometheusMetrics.h" #include "cache/GpuCacheMgr.h" -#include "server/Config.h" #include "metrics/SystemInfo.h" +#include "server/Config.h" #include "utils/Log.h" #include From d4e2be1cd8907fcd54fcbeb901c37b122e764377 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Thu, 14 Nov 2019 20:11:03 +0800 Subject: [PATCH 10/16] update --- core/src/metrics/prometheus/PrometheusMetrics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/metrics/prometheus/PrometheusMetrics.cpp b/core/src/metrics/prometheus/PrometheusMetrics.cpp index e55c01b5d0..19b2683280 100644 --- a/core/src/metrics/prometheus/PrometheusMetrics.cpp +++ b/core/src/metrics/prometheus/PrometheusMetrics.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "prometheus/PrometheusMetrics.h" +#include "metrics/prometheus/PrometheusMetrics.h" #include "cache/GpuCacheMgr.h" #include "metrics/SystemInfo.h" #include "server/Config.h" From 95ceb8c7ba67acab4ef2b861cf67026ca174bd0a Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Thu, 14 Nov 2019 20:22:49 +0800 Subject: [PATCH 11/16] memory usage increased slowly during searching vectors --- CHANGELOG.md | 1 + .../knowhere/adapter/SptagAdapter.cpp | 57 ++++++++++--------- .../index/knowhere/knowhere/common/Dataset.h | 16 ++++++ .../index/vector_index/IndexIDMAP.cpp | 35 ++++++------ .../knowhere/index/vector_index/IndexIVF.cpp | 32 +++++------ .../knowhere/index/vector_index/IndexNSG.cpp | 33 +++++------ core/src/index/unittest/test_kdt.cpp | 12 ++-- core/src/index/unittest/utils.cpp | 5 +- core/src/wrapper/VecImpl.cpp | 14 +++-- 9 files changed, 116 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb07803e74..988e177a2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ Please mark all change in change log and use the ticket from JIRA. # Milvus 0.6.0 (TODO) ## Bug +- \#228 - memory usage increased slowly during searching vectors - \#246 - Exclude src/external folder from code coverage for jenkin ci - \#248 - Reside src/external in thirdparty diff --git a/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp b/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp index b4c3910a01..db4a415261 100644 --- a/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp +++ b/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp @@ -89,34 +89,35 @@ ConvertToDataset(std::vector query_results) { } } - auto id_buf = MakeMutableBufferSmart((uint8_t*)p_id, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t*)p_dist, sizeof(float) * elems); - - // TODO: magic - std::vector id_bufs{nullptr, id_buf}; - std::vector dist_bufs{nullptr, dist_buf}; - - auto int64_type = std::make_shared(); - auto float_type = std::make_shared(); - - auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); - auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); - // auto id_array_data = std::make_shared(int64_type, sizeof(int64_t) * elems, id_bufs); - // auto dist_array_data = std::make_shared(float_type, sizeof(float) * elems, dist_bufs); - - // auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems); - // auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems); - - auto ids = std::make_shared>(id_array_data); - auto dists = std::make_shared>(dist_array_data); - std::vector array{ids, dists}; - - auto field_id = std::make_shared("id", std::make_shared()); - auto field_dist = std::make_shared("dist", std::make_shared()); - std::vector fields{field_id, field_dist}; - auto schema = std::make_shared(fields); - - return std::make_shared(array, schema); + // auto id_buf = MakeMutableBufferSmart((uint8_t*)p_id, sizeof(int64_t) * elems); + // auto dist_buf = MakeMutableBufferSmart((uint8_t*)p_dist, sizeof(float) * elems); + // + // // TODO: magic + // std::vector id_bufs{nullptr, id_buf}; + // std::vector dist_bufs{nullptr, dist_buf}; + // + // auto int64_type = std::make_shared(); + // auto float_type = std::make_shared(); + // + // auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); + // auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); + // // auto id_array_data = std::make_shared(int64_type, sizeof(int64_t) * elems, id_bufs); + // // auto dist_array_data = std::make_shared(float_type, sizeof(float) * elems, dist_bufs); + // + // // auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems); + // // auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems); + // + // auto ids = std::make_shared>(id_array_data); + // auto dists = std::make_shared>(dist_array_data); + // std::vector array{ids, dists}; + // + // auto field_id = std::make_shared("id", std::make_shared()); + // auto field_dist = std::make_shared("dist", std::make_shared()); + // std::vector fields{field_id, field_dist}; + // auto schema = std::make_shared(fields); + // + // return std::make_shared(array, schema); + return std::make_shared((void*)p_id, (void*)p_dist); } } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/common/Dataset.h b/core/src/index/knowhere/knowhere/common/Dataset.h index 1331239dd6..b101aba6a7 100644 --- a/core/src/index/knowhere/knowhere/common/Dataset.h +++ b/core/src/index/knowhere/knowhere/common/Dataset.h @@ -54,6 +54,9 @@ class Dataset { : tensor_(std::move(tensor)), tensor_schema_(std::move(tensor_schema)) { } + Dataset(void* ids, void* dists) : ids_(ids), dists_(dists) { + } + Dataset(const Dataset&) = delete; Dataset& operator=(const Dataset&) = delete; @@ -128,6 +131,16 @@ class Dataset { tensor_schema_ = std::move(tensor_schema); } + void* + ids() { + return ids_; + } + + void* + dist() { + return dists_; + } + // const Config & // meta() const { return meta_; } @@ -141,6 +154,9 @@ class Dataset { SchemaPtr array_schema_; std::vector tensor_; SchemaPtr tensor_schema_; + // TODO(yukun): using smart pointer + void* ids_; + void* dists_; // Config meta_; }; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index 98d25e5e5c..7aedf98613 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -80,23 +80,24 @@ IDMAP::Search(const DatasetPtr& dataset, const Config& config) { search_impl(rows, (float*)p_data, config->k, res_dis, res_ids, Config()); - auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); - - std::vector id_bufs{nullptr, id_buf}; - std::vector dist_bufs{nullptr, dist_buf}; - - auto int64_type = std::make_shared(); - auto float_type = std::make_shared(); - - auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); - auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); - - auto ids = std::make_shared>(id_array_data); - auto dists = std::make_shared>(dist_array_data); - std::vector array{ids, dists}; - - return std::make_shared(array, nullptr); + // auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); + // auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); + // + // std::vector id_bufs{nullptr, id_buf}; + // std::vector dist_bufs{nullptr, dist_buf}; + // + // auto int64_type = std::make_shared(); + // auto float_type = std::make_shared(); + // + // auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); + // auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); + // + // auto ids = std::make_shared>(id_array_data); + // auto dists = std::make_shared>(dist_array_data); + // std::vector array{ids, dists}; + // + // return std::make_shared(array, nullptr); + return std::make_shared((void*)res_ids, (void*)res_dis); } void diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index b2a2af29a8..7f30a97ea0 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -139,23 +139,23 @@ IVF::Search(const DatasetPtr& dataset, const Config& config) { // std::cout << ss_res_id.str() << std::endl; // std::cout << ss_res_dist.str() << std::endl << std::endl; - auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); + // auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); + // auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); + // + // std::vector id_bufs{nullptr, id_buf}; + // std::vector dist_bufs{nullptr, dist_buf}; + // + // auto int64_type = std::make_shared(); + // auto float_type = std::make_shared(); + // + // auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); + // auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); + // + // auto ids = std::make_shared>(id_array_data); + // auto dists = std::make_shared>(dist_array_data); + // std::vector array{ids, dists}; - std::vector id_bufs{nullptr, id_buf}; - std::vector dist_bufs{nullptr, dist_buf}; - - auto int64_type = std::make_shared(); - auto float_type = std::make_shared(); - - auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); - auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); - - auto ids = std::make_shared>(id_array_data); - auto dists = std::make_shared>(dist_array_data); - std::vector array{ids, dists}; - - return std::make_shared(array, nullptr); + return std::make_shared((void*)res_ids, (void*)res_dis); } void diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp index 8f6d93d7ff..204819517a 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp @@ -88,23 +88,24 @@ NSG::Search(const DatasetPtr& dataset, const Config& config) { s_params.search_length = build_cfg->search_length; index_->Search((float*)p_data, rows, dim, build_cfg->k, res_dis, res_ids, s_params); - auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); + // auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); + // auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); - std::vector id_bufs{nullptr, id_buf}; - std::vector dist_bufs{nullptr, dist_buf}; - - auto int64_type = std::make_shared(); - auto float_type = std::make_shared(); - - auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); - auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); - - auto ids = std::make_shared>(id_array_data); - auto dists = std::make_shared>(dist_array_data); - std::vector array{ids, dists}; - - return std::make_shared(array, nullptr); + // std::vector id_bufs{nullptr, id_buf}; + // std::vector dist_bufs{nullptr, dist_buf}; + // + // auto int64_type = std::make_shared(); + // auto float_type = std::make_shared(); + // + // auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); + // auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); + // + // auto ids = std::make_shared>(id_array_data); + // auto dists = std::make_shared>(dist_array_data); + // std::vector array{ids, dists}; + // + // return std::make_shared(array, nullptr); + return std::make_shared((void*)res_ids, (void*)res_dis); } IndexModelPtr diff --git a/core/src/index/unittest/test_kdt.cpp b/core/src/index/unittest/test_kdt.cpp index 5400881875..bbc7dcf94c 100644 --- a/core/src/index/unittest/test_kdt.cpp +++ b/core/src/index/unittest/test_kdt.cpp @@ -66,15 +66,19 @@ TEST_F(KDTTest, kdt_basic) { AssertAnns(result, nq, k); { - auto ids = result->array()[0]; - auto dists = result->array()[1]; + // auto ids = result->array()[0]; + // auto dists = result->array()[1]; + auto ids = result->ids(); + auto dists = result->dist(); std::stringstream ss_id; std::stringstream ss_dist; for (auto i = 0; i < nq; i++) { for (auto j = 0; j < k; ++j) { - ss_id << *ids->data()->GetValues(1, i * k + j) << " "; - ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; + ss_id << *((int64_t*)(ids) + i * k + j) << " "; + ss_dist << *((float*)(dists) + i * k + j) << " "; + // ss_id << *ids->data()->GetValues(1, i * k + j) << " "; + // ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; } ss_id << std::endl; ss_dist << std::endl; diff --git a/core/src/index/unittest/utils.cpp b/core/src/index/unittest/utils.cpp index d4a59bafbb..2556b60fad 100644 --- a/core/src/index/unittest/utils.cpp +++ b/core/src/index/unittest/utils.cpp @@ -151,9 +151,10 @@ generate_query_dataset(int64_t nb, int64_t dim, float* xb) { void AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) { - auto ids = result->array()[0]; + auto ids = result->ids(); for (auto i = 0; i < nq; i++) { - EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); + EXPECT_EQ(i, *((int64_t*)(ids) + i * k)); + // EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); } } diff --git a/core/src/wrapper/VecImpl.cpp b/core/src/wrapper/VecImpl.cpp index 05293b53f1..e7967cbf59 100644 --- a/core/src/wrapper/VecImpl.cpp +++ b/core/src/wrapper/VecImpl.cpp @@ -84,8 +84,8 @@ VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* i Config search_cfg = cfg; auto res = index_->Search(dataset, search_cfg); - auto ids_array = res->array()[0]; - auto dis_array = res->array()[1]; + // auto ids_array = res->array()[0]; + // auto dis_array = res->array()[1]; //{ // auto& ids = ids_array; @@ -104,12 +104,14 @@ VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* i // std::cout << "dist\n" << ss_dist.str() << std::endl; //} - auto p_ids = ids_array->data()->GetValues(1, 0); - auto p_dist = dis_array->data()->GetValues(1, 0); + // auto p_ids = ids_array->data()->GetValues(1, 0); + // auto p_dist = dis_array->data()->GetValues(1, 0); // TODO(linxj): avoid copy here. - memcpy(ids, p_ids, sizeof(int64_t) * nq * k); - memcpy(dist, p_dist, sizeof(float) * nq * k); + memcpy(ids, res->ids(), sizeof(int64_t) * nq * k); + memcpy(dist, res->dist(), sizeof(float) * nq * k); + free(res->ids()); + free(res->dist()); } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); From 13cb3b89140b6cd9c092d19971a1c5e675e8423f Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Thu, 14 Nov 2019 21:02:23 +0800 Subject: [PATCH 12/16] Update CMakeLists.txt --- core/unittest/CMakeLists.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 82aaa9efd1..401fd4ea3b 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -97,7 +97,14 @@ set(common_files ${helper_files} ) -set(unittest_libs +if (MILVUS_WITH_PROMETHEUS) + set(unittest_libs + prometheus-cpp-push + prometheus-cpp-pull + prometheus-cpp-core + ) +endif () +set(unittest_libs ${unittest_libs} sqlite libboost_system.a libboost_filesystem.a @@ -113,13 +120,6 @@ set(unittest_libs dl z ) -if (MILVUS_WITH_PROMETHEUS) - set(unittest_libs ${unittest_libs} - prometheus-cpp-push - prometheus-cpp-pull - prometheus-cpp-core - ) -endif () if (MILVUS_GPU_VERSION) include_directories("${CUDA_INCLUDE_DIRS}") @@ -139,4 +139,4 @@ add_subdirectory(db) add_subdirectory(wrapper) add_subdirectory(metrics) add_subdirectory(scheduler) -add_subdirectory(server) \ No newline at end of file +add_subdirectory(server) From 8ae3a839481a7c156aefb3fab2b9d873b52f3ad4 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Thu, 14 Nov 2019 21:12:51 +0800 Subject: [PATCH 13/16] Update CMakeLists.txt --- core/unittest/CMakeLists.txt | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 401fd4ea3b..01e1054f7e 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -97,14 +97,7 @@ set(common_files ${helper_files} ) -if (MILVUS_WITH_PROMETHEUS) - set(unittest_libs - prometheus-cpp-push - prometheus-cpp-pull - prometheus-cpp-core - ) -endif () -set(unittest_libs ${unittest_libs} +set(unittest_libs sqlite libboost_system.a libboost_filesystem.a @@ -117,9 +110,18 @@ set(unittest_libs ${unittest_libs} pthread metrics gfortran - dl - z ) +if (MILVUS_WITH_PROMETHEUS) + set(unittest_libs ${unittest_libs} + prometheus-cpp-push + prometheus-cpp-pull + prometheus-cpp-core + ) +endif () +set(unittest_libs ${unittest_libs} + dl + z + ) if (MILVUS_GPU_VERSION) include_directories("${CUDA_INCLUDE_DIRS}") From 2306260ba66be93ef990dfc9a696b56ca66548fe Mon Sep 17 00:00:00 2001 From: zhenwu Date: Fri, 15 Nov 2019 09:47:22 +0800 Subject: [PATCH 14/16] Update server_version --- tests/milvus-java-test/pom.xml | 4 ++-- tests/milvus_python_test/test_ping.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/milvus-java-test/pom.xml b/tests/milvus-java-test/pom.xml index 4da715e292..bff6f1de61 100644 --- a/tests/milvus-java-test/pom.xml +++ b/tests/milvus-java-test/pom.xml @@ -99,7 +99,7 @@ io.milvus milvus-sdk-java - 0.2.0-SNAPSHOT + 0.3.0 @@ -134,4 +134,4 @@ - \ No newline at end of file + diff --git a/tests/milvus_python_test/test_ping.py b/tests/milvus_python_test/test_ping.py index d63ab93f11..3831dcd6ac 100644 --- a/tests/milvus_python_test/test_ping.py +++ b/tests/milvus_python_test/test_ping.py @@ -1,7 +1,7 @@ import logging import pytest -__version__ = '0.5.1' +__version__ = '0.6.0' class TestPing: From 4840aa33b3ef207b463937ba969243540aa725e4 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Fri, 15 Nov 2019 10:11:55 +0800 Subject: [PATCH 15/16] fix test_ivf bug --- core/src/index/unittest/test_ivf.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/src/index/unittest/test_ivf.cpp b/core/src/index/unittest/test_ivf.cpp index ae1034b9e0..5cb820df95 100644 --- a/core/src/index/unittest/test_ivf.cpp +++ b/core/src/index/unittest/test_ivf.cpp @@ -181,11 +181,13 @@ TEST_P(IVFTest, clone_test) { // PrintResult(result, nq, k); auto AssertEqual = [&](knowhere::DatasetPtr p1, knowhere::DatasetPtr p2) { - auto ids_p1 = p1->array()[0]; - auto ids_p2 = p2->array()[0]; + auto ids_p1 = p1->ids(); + auto ids_p2 = p2->ids(); for (int i = 0; i < nq * k; ++i) { - EXPECT_EQ(*(ids_p2->data()->GetValues(1, i)), *(ids_p1->data()->GetValues(1, i))); + EXPECT_EQ(*((int64_t*)(ids_p2) + i), *((int64_t*)(ids_p1) + i)); + // EXPECT_EQ(*(ids_p2->data()->GetValues(1, i)), *(ids_p1->data()->GetValues(1, + // i))); } }; From 822b121811459466ccba9e9f4d9fe07e6aac30f7 Mon Sep 17 00:00:00 2001 From: zhenwu Date: Fri, 15 Nov 2019 10:14:56 +0800 Subject: [PATCH 16/16] Revert docker directory --- docker/deploy/cpu_version/Dockerfile | 18 ------------------ .../ubuntu18.04/docker-entrypoint.sh | 10 ---------- .../{gpu_version => }/ubuntu16.04/Dockerfile | 0 .../docker-entrypoint.sh | 0 .../{gpu_version => }/ubuntu18.04/Dockerfile | 0 .../docker-entrypoint.sh | 0 6 files changed, 28 deletions(-) delete mode 100644 docker/deploy/cpu_version/Dockerfile delete mode 100755 docker/deploy/gpu_version/ubuntu18.04/docker-entrypoint.sh rename docker/deploy/{gpu_version => }/ubuntu16.04/Dockerfile (100%) rename docker/deploy/{cpu_version => ubuntu16.04}/docker-entrypoint.sh (100%) rename docker/deploy/{gpu_version => }/ubuntu18.04/Dockerfile (100%) rename docker/deploy/{gpu_version/ubuntu16.04 => ubuntu18.04}/docker-entrypoint.sh (100%) diff --git a/docker/deploy/cpu_version/Dockerfile b/docker/deploy/cpu_version/Dockerfile deleted file mode 100644 index 317001385e..0000000000 --- a/docker/deploy/cpu_version/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - gfortran libsqlite3-dev libmysqlclient-dev && \ - apt-get remove --purge -y && \ - rm -rf /var/lib/apt/lists/* - -RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so - -COPY ./docker-entrypoint.sh /opt -COPY ./milvus /opt/milvus -ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/milvus/lib" - -ENTRYPOINT [ "/opt/docker-entrypoint.sh" ] - -CMD [ "start" ] - -EXPOSE 19530 diff --git a/docker/deploy/gpu_version/ubuntu18.04/docker-entrypoint.sh b/docker/deploy/gpu_version/ubuntu18.04/docker-entrypoint.sh deleted file mode 100755 index 446c174d74..0000000000 --- a/docker/deploy/gpu_version/ubuntu18.04/docker-entrypoint.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -e - -if [ "$1" == 'start' ]; then - cd /opt/milvus/scripts && ./start_server.sh -fi - -exec "$@" - diff --git a/docker/deploy/gpu_version/ubuntu16.04/Dockerfile b/docker/deploy/ubuntu16.04/Dockerfile similarity index 100% rename from docker/deploy/gpu_version/ubuntu16.04/Dockerfile rename to docker/deploy/ubuntu16.04/Dockerfile diff --git a/docker/deploy/cpu_version/docker-entrypoint.sh b/docker/deploy/ubuntu16.04/docker-entrypoint.sh similarity index 100% rename from docker/deploy/cpu_version/docker-entrypoint.sh rename to docker/deploy/ubuntu16.04/docker-entrypoint.sh diff --git a/docker/deploy/gpu_version/ubuntu18.04/Dockerfile b/docker/deploy/ubuntu18.04/Dockerfile similarity index 100% rename from docker/deploy/gpu_version/ubuntu18.04/Dockerfile rename to docker/deploy/ubuntu18.04/Dockerfile diff --git a/docker/deploy/gpu_version/ubuntu16.04/docker-entrypoint.sh b/docker/deploy/ubuntu18.04/docker-entrypoint.sh similarity index 100% rename from docker/deploy/gpu_version/ubuntu16.04/docker-entrypoint.sh rename to docker/deploy/ubuntu18.04/docker-entrypoint.sh