From e852816176407de1c7335fbf7ff2b2b4a2da2513 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Wed, 18 Sep 2019 15:34:47 +0800 Subject: [PATCH 01/16] fix memory leak Former-commit-id: 397a13f633b1b120c516db97ed46e5c8836db1ce --- cpp/src/metrics/SystemInfo.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/metrics/SystemInfo.cpp b/cpp/src/metrics/SystemInfo.cpp index 61ae42c438..5db2448ee9 100644 --- a/cpp/src/metrics/SystemInfo.cpp +++ b/cpp/src/metrics/SystemInfo.cpp @@ -245,6 +245,7 @@ SystemInfo::CPUTemperature(){ float temp; fscanf(file, "%f", &temp); result.push_back(temp / 1000); + fclose(file); } } From e8224fc61dcd20f2595ed899fb2acdf019ced005 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Wed, 18 Sep 2019 19:53:08 +0800 Subject: [PATCH 02/16] MS-569 Complete the NOTICE.md Former-commit-id: dd04582d0ac67e31fdb503a70647cdd3eadebd4a --- cpp/CHANGELOG.md | 1 + cpp/NOTICE.md | 34 +++++++++++++++++----------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 886b383cd8..1327fee901 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -20,6 +20,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-554 - Change license to Apache 2.0 - MS-561 - Add contributing guidelines, code of conduct and README docs - MS-567 - Add NOTICE.md +- MS-569 - Complete the NOTICE.md # Milvus 0.4.0 (2019-09-12) diff --git a/cpp/NOTICE.md b/cpp/NOTICE.md index 970dd054c0..57766a77db 100644 --- a/cpp/NOTICE.md +++ b/cpp/NOTICE.md @@ -6,24 +6,24 @@ | Name | License | | ------------- | ------------------------------------------------------------ | -| Apache Arrow | Apache License 2.0 | -| Boost | Boost Software License | -| BZIP2 | BSD-like license | -| FAISS | MIT | -| Gtest | BSD 3-Clause | -| LAPACK | BSD 3-Clause | +| Apache Arrow | [Apache License 2.0](https://github.com/apache/arrow/blob/master/LICENSE.txt) | +| Boost | [Boost Software License](https://github.com/boostorg/boost/blob/master/LICENSE_1_0.txt) | +| BZip2 | [BZip2](http://www.bzip.org/) | +| FAISS | [MIT](https://github.com/facebookresearch/faiss/blob/master/LICENSE) | +| Gtest | [BSD 3-Clause](https://github.com/google/googletest/blob/master/LICENSE) | +| LAPACK | [LAPACK](https://github.com/Reference-LAPACK/lapack/blob/master/LICENSE) | | LZ4 | [BSD 2-Clause](https://github.com/Blosc/c-blosc/blob/master/LICENSES/LZ4.txt) | -| MySQLPP | LGPL 2.1 | -| OpenBLAS | BSD 3-Clause | -| Prometheus | Apache License 2.0 | -| Snappy | [BSD](https://github.com/Blosc/c-blosc/blob/master/LICENSES/SNAPPY.txt) | +| MySQLPP | [LGPL 2.1](https://tangentsoft.com/mysqlpp/artifact/b128a66dab867923) | +| OpenBLAS | [BSD 3-Clause](https://github.com/xianyi/OpenBLAS/blob/develop/LICENSE) | +| Prometheus | [Apache License 2.0](https://github.com/prometheus/prometheus/blob/master/LICENSE) | +| Snappy | [BSD](https://github.com/google/snappy/blob/master/COPYING) | | SQLite | [Public Domain](https://www.sqlite.org/copyright.html) | -| SQLite-ORM | BSD 3-Clause | -| yaml-cpp | MIT | -| ZLIB | [zlib](http://zlib.net/zlib_license.html) | +| SQLite-ORM | [BSD 3-Clause](https://github.com/fnc12/sqlite_orm/blob/master/LICENSE) | +| yaml-cpp | [MIT](https://github.com/jbeder/yaml-cpp/blob/master/LICENSE) | +| ZLIB | [ZLIB](http://zlib.net/zlib_license.html) | | ZSTD | [BSD](https://github.com/facebook/zstd/blob/dev/LICENSE) | -| libunwind | MIT | -| gperftools | BSD 3-Clause | -| gRPC | Apache 2.0 | -| EASYLOGGINGPP | MIT | +| libunwind | [MIT](https://github.com/libunwind/libunwind/blob/master/LICENSE) | +| gperftools | [BSD 3-Clause](https://github.com/gperftools/gperftools/blob/master/COPYING) | +| grpc | [Apache 2.0](https://github.com/grpc/grpc/blob/master/LICENSE) | +| EASYLOGGINGPP | [MIT](https://github.com/zuhd-org/easyloggingpp/blob/master/LICENSEhttps://github.com/zuhd-org/easyloggingpp/blob/master/LICENSE) | From 8382fa623c5fc4507e56d11e104859325c5aa16b Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Wed, 18 Sep 2019 20:03:01 +0800 Subject: [PATCH 03/16] MS-570 - Add prometheus docker-compose file Former-commit-id: 1b60196354afab6e8fd3355d0752d66cf9722702 --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index c5d27e3461..a1b06bba14 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -136,6 +136,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-523 - Config file validation - MS-539 - Remove old task code - MS-546 - Add simple mode resource_config +- MS-570 - Add prometheus docker-compose file ## New Feature - MS-343 - Implement ResourceMgr From 1a3231e6f6e4f0343180397e395cb277f74483a8 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Wed, 18 Sep 2019 20:03:19 +0800 Subject: [PATCH 04/16] MS-570 - Add prometheus docker-compose file Former-commit-id: a81bcfd940892284d9f0c26049e4c42c0cf80f34 --- docker/alertmanager.yml | 19 +++++++++++ docker/docker-compose-monitor.yml | 56 +++++++++++++++++++++++++++++++ docker/prometheus.yml | 46 +++++++++++++++++++++++++ docker/server_down.yml | 8 +++++ 4 files changed, 129 insertions(+) create mode 100644 docker/alertmanager.yml create mode 100644 docker/docker-compose-monitor.yml create mode 100644 docker/prometheus.yml create mode 100644 docker/server_down.yml diff --git a/docker/alertmanager.yml b/docker/alertmanager.yml new file mode 100644 index 0000000000..e1f079dce0 --- /dev/null +++ b/docker/alertmanager.yml @@ -0,0 +1,19 @@ +global: + resolve_timeout: 5m + +route: + group_by: ['alertname'] + group_wait: 10s + group_interval: 10s + repeat_interval: 1h + receiver: 'web.hook' +receivers: +- name: 'web.hook' + webhook_configs: + - url: 'http://127.0.0.1:5001/' +inhibit_rules: + - source_match: + severity: 'critical' + target_match: + severity: 'warning' + equal: ['alertname', 'dev', 'instance'] diff --git a/docker/docker-compose-monitor.yml b/docker/docker-compose-monitor.yml new file mode 100644 index 0000000000..9cb1fb1902 --- /dev/null +++ b/docker/docker-compose-monitor.yml @@ -0,0 +1,56 @@ +version: '2.3' + +networks: + monitor: + driver: bridge + +services: + prometheus: + image: prom/prometheus:v2.11.1 + container_name: prometheus + hostname: prometheus + restart: always + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - ./server_down.yml:/etc/prometheus/node_down.yml + ports: + - "9090:9090" + networks: + - monitor + + alertmanager: + image: prom/alertmanager + container_name: alertmanager + hostname: alertmanager + restart: always + volumes: + - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml + ports: + - "9093:9093" + networks: + - monitor + + grafana: + image: grafana/grafana + container_name: grafana + hostname: grafana + restart: always + ports: + - "3000:3000" + networks: + - monitor + + milvus: + runtime: nvidia + image: registry.zilliz.com/milvus/engine:branch-0.4.0-release + container_name: milvus + hostname: milvus + restart: always + volumes: + - ../cpp/conf/server_config.yaml:/opt/milvus/cpp/conf/server_config.yaml + - ../cpp/conf/log_config.conf:/opt/milvus/cpp/conf/log_config.conf + ports: + - "8080:8080" + - "19530:19530" + networks: + - monitor diff --git a/docker/prometheus.yml b/docker/prometheus.yml new file mode 100644 index 0000000000..7a2a07b390 --- /dev/null +++ b/docker/prometheus.yml @@ -0,0 +1,46 @@ +# my global config +global: + scrape_interval: 15s # Set the scrape interval to every 1 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: ['localhost:9093'] + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + - "serverdown.yml" # add alerting rules + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + static_configs: + - targets: ['localhost:9090'] + + # scrape metrics of server + - job_name: 'milvus_server' + scrape_interval: 1s + static_configs: + - targets: ['localhost:8080'] + + # scrape metrics of server + - job_name: 'milvus_server_1' + scrape_interval: 1s + static_configs: + - targets: ['localhost:8080'] + + + + # under development + - job_name: 'pushgateway' + static_configs: + - targets: ['localhost:9091'] diff --git a/docker/server_down.yml b/docker/server_down.yml new file mode 100644 index 0000000000..8a7077c7f8 --- /dev/null +++ b/docker/server_down.yml @@ -0,0 +1,8 @@ +groups: +- name: milvus + rules: + - alert: MilvusServerDown + expr: up{job="milvus_server"} + for: 1s + labels: + serverity: page From 3978c2a71f73397a15ccc465ed1f339c6e3fa08a Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Thu, 19 Sep 2019 11:04:56 +0800 Subject: [PATCH 05/16] add Log.h in knowhere Former-commit-id: e866a09e487f1ef60816bd0ee6c7308e4bf1ff7e --- cpp/src/core/knowhere/knowhere/common/Log.h | 35 +++++++++++++++++++ .../knowhere/knowhere/common/exception.cpp | 1 + .../knowhere/index/vector_index/gpu_ivf.cpp | 2 ++ 3 files changed, 38 insertions(+) create mode 100644 cpp/src/core/knowhere/knowhere/common/Log.h diff --git a/cpp/src/core/knowhere/knowhere/common/Log.h b/cpp/src/core/knowhere/knowhere/common/Log.h new file mode 100644 index 0000000000..0d99c624ab --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/common/Log.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "utils/easylogging++.h" + +namespace zilliz { +namespace knowhere { +#define KNOWHERE_DOMAIN_NAME "[KNOWHERE] " +#define KNOWHERE_ERROR_TEXT "KNOWHERE Error:" + +#define KNOWHERE_LOG_TRACE LOG(TRACE) << KNOWHERE_DOMAIN_NAME +#define KNOWHERE_LOG_DEBUG LOG(DEBUG) << KNOWHERE_DOMAIN_NAME +#define KNOWHERE_LOG_INFO LOG(INFO) << KNOWHERE_DOMAIN_NAME +#define KNOWHERE_LOG_WARNING LOG(WARNING) << KNOWHERE_DOMAIN_NAME +#define KNOWHERE_LOG_ERROR LOG(ERROR) << KNOWHERE_DOMAIN_NAME +#define KNOWHERE_LOG_FATAL LOG(FATAL) << KNOWHERE_DOMAIN_NAME +} // namespace knowhere +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/core/knowhere/knowhere/common/exception.cpp b/cpp/src/core/knowhere/knowhere/common/exception.cpp index fcf3886d9b..eb19e21bb0 100644 --- a/cpp/src/core/knowhere/knowhere/common/exception.cpp +++ b/cpp/src/core/knowhere/knowhere/common/exception.cpp @@ -19,6 +19,7 @@ #include #include "exception.h" +#include "Log.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp index fd04501c83..51fe237658 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp @@ -30,6 +30,7 @@ #include "cloner.h" #include "knowhere/adapter/faiss_adopt.h" #include "gpu_ivf.h" +#include "knowhere/common/Log.h" #include @@ -223,6 +224,7 @@ VectorIndexPtr GPUIVFPQ::CopyGpuToCpu(const Config &config) { } IndexModelPtr GPUIVFSQ::Train(const DatasetPtr &dataset, const Config &config) { + KNOWHERE_LOG_WARNING << "lalalalalalala"; auto nlist = config["nlist"].as(); auto nbits = config["nbits"].as(); // TODO(linxj): gpu only support SQ4 SQ8 SQ16 gpu_id_ = config.get_with_default("gpu_id", gpu_id_); From 5c16730469489982b30af86ca83fa189cc730f73 Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 19 Sep 2019 12:52:46 +0800 Subject: [PATCH 06/16] fix cmake Former-commit-id: dd8288d065c48eccc6dff699d153e48fbe5205f4 --- cpp/src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 3bf20ac2fd..6ee3966c4f 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -152,7 +152,7 @@ if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") endif () cuda_add_library(milvus_engine STATIC ${engine_files}) -target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs}) +target_link_libraries(milvus_engine ${engine_libs} knowhere ${third_party_libs}) add_library(metrics STATIC ${metrics_files}) @@ -180,7 +180,7 @@ add_executable(milvus_server ${utils_files} ) -target_link_libraries(milvus_server ${server_libs} knowhere ${third_party_libs}) +target_link_libraries(milvus_server ${server_libs}) install(TARGETS milvus_server DESTINATION bin) From 4b6be7b0c09fc4555267e6e5a0e6858ab81779e3 Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Thu, 19 Sep 2019 14:10:22 +0800 Subject: [PATCH 07/16] Update README.md Former-commit-id: aeac98a8cdc3cf186ee9c3e8916ca646345e6218 --- cpp/README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index 18c1131f10..d32f04f316 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -4,9 +4,9 @@ Firstly, welcome, and thanks for your interest in [Milvus](https://milvus.io)! N ## What is Milvus -Milvus is an open source vector search engine that makes incredibly fast querying speed enhancement over current solutions of massive vector processing. Built on optimized indexing algorithm, it is compatible with major AI/ML models. +Milvus is an open source vector search engine that supports similarity search of large-scale vectors. Built on optimized indexing algorithm, it is compatible with major AI/ML models. -Milvus was developed by researchers and engineers in ZILLIZ, a tech startup that intends to reinvent data science, with the purpose of providing enterprises with efficient and scalable similarity search and analysis of feature vectors and unstructured data. +Milvus was developed by ZILLIZ, a tech startup that intends to reinvent data science, with the purpose of providing enterprises with efficient and scalable similarity search and analysis of feature vectors and unstructured data. Milvus provides stable Python and C++ APIs, as well as RESTful API. @@ -182,11 +182,11 @@ $ python3 example.py Contributions are welcomed and greatly appreciated. If you want to contribute to Milvus, please read the [contribution guidelines](CONTRIBUTING.md). This project adheres to the [code of conduct](CODE OF CONDUCT.md) of Milvus. By participating, you are expected to uphold this code. -We use [GitHub issues] to track issues and bugs. For general questions and discussions, please go to [Milvus Forum]. +We use [GitHub issues](https://github.com/milvus-io/milvus/issues) to track issues and bugs. For general questions and discussions, please go to [Milvus Forum]. ## Join the Milvus community -For public discussion of Milvus, please join our [discussion group]. +For public discussion of Milvus, please join our [discussion group](milvusio.slack.com). ## Milvus Roadmap @@ -194,7 +194,7 @@ Please read our [roadmap](milvus-io/milvus/docs/master/roadmap.md) to learn abou ## Resources -[Milvus official website](https://milvus.io) +[Milvus official website](https://www.milvus.io) [Milvus docs](https://www.milvus.io/docs/en/QuickStart/) @@ -204,7 +204,6 @@ Please read our [roadmap](milvus-io/milvus/docs/master/roadmap.md) to learn abou [Milvus roadmap](https://www.milvus-io/docs/master/roadmap.md) -[Milvus white paper] ## License From c89a99b8002866822c64c58daa427bd872bed2bd Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 19 Sep 2019 09:29:21 +0800 Subject: [PATCH 08/16] MS-572 fix Milvus crash when get SIGINT Former-commit-id: 32b92068024731fcc3451271c7f66f8a37b68cbd --- cpp/src/server/Server.cpp | 15 +++++++--- cpp/src/server/Server.h | 15 +++++++--- cpp/src/server/grpc_impl/GrpcMilvusServer.cpp | 28 ++++++++++++------- cpp/src/server/grpc_impl/GrpcMilvusServer.h | 25 +++++++++++++---- cpp/src/utils/SignalUtil.cpp | 4 +-- 5 files changed, 62 insertions(+), 25 deletions(-) diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 9920b30246..e4cb7b8d4a 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -42,7 +42,7 @@ namespace zilliz { namespace milvus { namespace server { -Server& +Server & Server::Instance() { static Server server; return server; @@ -203,6 +203,10 @@ Server::Start() { std::cout << "Milvus server start successfully." << std::endl; StartService(); + while (running_) { + sleep(10); + } + } catch (std::exception &ex) { std::cerr << "Milvus server encounter exception: " << std::string(ex.what()) << "Is another server instance running?"; @@ -210,7 +214,6 @@ Server::Start() { } } while (false); - Stop(); return 0; } @@ -242,6 +245,10 @@ Server::Stop() { } StopService(); + { + std::lock_guard lock(mutex_); + running_ = false; + } std::cerr << "Milvus server is closed!" << std::endl; } @@ -264,12 +271,12 @@ Server::StartService() { engine::KnowhereResource::Initialize(); engine::StartSchedulerService(); DBWrapper::GetInstance().StartService(); - grpc::GrpcMilvusServer::StartService(); + grpc::GrpcMilvusServer::GetInstance().Start(); } void Server::StopService() { - grpc::GrpcMilvusServer::StopService(); + grpc::GrpcMilvusServer::GetInstance().Stop(); DBWrapper::GetInstance().StopService(); engine::StopSchedulerService(); engine::KnowhereResource::Finalize(); diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index eaf19d6b15..a4757cd0c6 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -21,23 +21,27 @@ #include #include +#include + namespace zilliz { namespace milvus { namespace server { class Server { -public: + public: static Server &Instance(); - void Init(int64_t daemonized, const std::string &pid_filename, const std::string &config_filename, + void Init(int64_t daemonized, + const std::string &pid_filename, + const std::string &config_filename, const std::string &log_config_file); int Start(); void Stop(); -private: + private: Server(); ~Server(); @@ -50,12 +54,15 @@ private: void StopService(); -private: + private: int64_t daemonized_ = 0; int pid_fd = -1; std::string pid_filename_; std::string config_filename_; std::string log_config_file_; + + std::mutex mutex_; + bool running_ = true; }; // Server } // server diff --git a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp b/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp index 6770e3111a..1af3557488 100644 --- a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp @@ -42,7 +42,6 @@ namespace milvus { namespace server { namespace grpc { -static std::unique_ptr<::grpc::Server> server; constexpr long MESSAGE_SIZE = -1; @@ -58,13 +57,22 @@ class NoReusePortOption : public ::grpc::ServerBuilderOption { }; +void +GrpcMilvusServer::Start() { + thread_ptr_ = std::make_shared(&GrpcMilvusServer::StartService, this); +} + +void +GrpcMilvusServer::Stop() { + StopService(); + if (thread_ptr_) { + thread_ptr_->join(); + thread_ptr_ = nullptr; + } +} + Status GrpcMilvusServer::StartService() { - if (server != nullptr) { - std::cout << "stop service!\n"; - StopService(); - } - ServerConfig &config = ServerConfig::GetInstance(); ConfigNode server_config = config.GetConfig(CONFIG_SERVER); ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); @@ -87,16 +95,16 @@ GrpcMilvusServer::StartService() { builder.AddListeningPort(server_address, ::grpc::InsecureServerCredentials()); builder.RegisterService(&service); - server = builder.BuildAndStart(); - server->Wait(); + server_ptr_ = builder.BuildAndStart(); + server_ptr_->Wait(); return Status::OK(); } Status GrpcMilvusServer::StopService() { - if (server != nullptr) { - server->Shutdown(); + if (server_ptr_ != nullptr) { + server_ptr_->Shutdown(); } return Status::OK(); diff --git a/cpp/src/server/grpc_impl/GrpcMilvusServer.h b/cpp/src/server/grpc_impl/GrpcMilvusServer.h index 751a2cd2a3..40df193a2f 100644 --- a/cpp/src/server/grpc_impl/GrpcMilvusServer.h +++ b/cpp/src/server/grpc_impl/GrpcMilvusServer.h @@ -21,6 +21,8 @@ #include #include +#include +#include namespace zilliz { namespace milvus { @@ -28,12 +30,25 @@ namespace server { namespace grpc { class GrpcMilvusServer { -public: - static Status - StartService(); + public: + static GrpcMilvusServer& GetInstance() { + static GrpcMilvusServer grpc_server; + return grpc_server; + } - static Status - StopService(); + void Start(); + void Stop(); + + private: + GrpcMilvusServer() = default; + ~GrpcMilvusServer() = default; + + Status StartService(); + Status StopService(); + + private: + std::unique_ptr<::grpc::Server> server_ptr_; + std::shared_ptr thread_ptr_; }; } diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index e380dc6c85..4da8d31eb0 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -31,7 +31,7 @@ void SignalUtil::HandleSignal(int signum){ switch(signum){ case SIGINT: case SIGUSR2:{ - SERVER_LOG_INFO << "Server received signal:" << std::to_string(signum); + SERVER_LOG_INFO << "Server received signal: " << signum; server::Server& server_ptr = server::Server::Instance(); server_ptr.Stop(); @@ -39,7 +39,7 @@ void SignalUtil::HandleSignal(int signum){ exit(0); } default:{ - SERVER_LOG_INFO << "Server received critical signal:" << std::to_string(signum); + SERVER_LOG_INFO << "Server received critical signal: " << signum; SignalUtil::PrintStacktrace(); server::Server& server_ptr = server::Server::Instance(); From 4d23ecf082b06a6b19e5e053af22adc792e8c3e4 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 19 Sep 2019 14:15:56 +0800 Subject: [PATCH 09/16] MS-572 re-org signal handle Former-commit-id: a357dd78d214c3f2c39538ea32f7fc492d72b6f2 --- cpp/CHANGELOG.md | 1 + cpp/src/main.cpp | 54 ++++----- cpp/src/server/Server.cpp | 109 ++++++++---------- cpp/src/server/Server.h | 8 +- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 2 +- .../{GrpcMilvusServer.cpp => GrpcServer.cpp} | 18 +-- .../{GrpcMilvusServer.h => GrpcServer.h} | 11 +- cpp/src/utils/SignalUtil.cpp | 18 +-- 8 files changed, 101 insertions(+), 120 deletions(-) rename cpp/src/server/grpc_impl/{GrpcMilvusServer.cpp => GrpcServer.cpp} (90%) rename cpp/src/server/grpc_impl/{GrpcMilvusServer.h => GrpcServer.h} (87%) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index d4d609d4aa..300d40f0af 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -5,6 +5,7 @@ Please mark all change in change log and use the ticket from JIRA. # Milvus 0.5.0 (TODO) ## Bug +- MS-572 - Milvus crash when get SIGINT ## Improvement - MS-552 - Add and change the easylogging library diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 3065e7d684..5819463d60 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -15,19 +15,20 @@ // specific language governing permissions and limitations // under the License. -#include "server/Server.h" -#include "version.h" - #include #include #include #include #include -#include "utils/easylogging++.h" -#include "metrics/Metrics.h" +#include +#include "utils/easylogging++.h" #include "utils/SignalUtil.h" #include "utils/CommonUtil.h" +#include "metrics/Metrics.h" +#include "server/Server.h" +#include "version.h" + INITIALIZE_EASYLOGGINGPP @@ -40,12 +41,6 @@ main(int argc, char *argv[]) { std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl; std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << " built at " << BUILD_TIME << std::endl; - signal(SIGINT, server::SignalUtil::HandleSignal); - signal(SIGSEGV, server::SignalUtil::HandleSignal); - signal(SIGUSR1, server::SignalUtil::HandleSignal); - signal(SIGUSR2, server::SignalUtil::HandleSignal); - - std::string app_name = basename(argv[0]); static struct option long_options[] = {{"conf_file", required_argument, 0, 'c'}, {"log_conf_file", required_argument, 0, 'l'}, {"help", no_argument, 0, 'h'}, @@ -55,14 +50,12 @@ main(int argc, char *argv[]) { int option_index = 0; int64_t start_daemonized = 0; -// int pid_fd; std::string config_filename, log_config_file; std::string pid_filename; + std::string app_name = argv[0]; - app_name = argv[0]; - - if(argc < 2) { + if (argc < 2) { print_help(app_name); std::cout << "Milvus server exit..." << std::endl; return EXIT_FAILURE; @@ -94,29 +87,38 @@ main(int argc, char *argv[]) { break; } - case 'd': - start_daemonized = 1; + case 'd':start_daemonized = 1; break; - case 'h': - print_help(app_name); + case 'h':print_help(app_name); return EXIT_SUCCESS; - case '?': - print_help(app_name); + case '?':print_help(app_name); return EXIT_FAILURE; - default: - print_help(app_name); + default:print_help(app_name); break; } } - server::Server& server = server::Server::Instance(); + server::Server &server = server::Server::Instance(); server.Init(start_daemonized, pid_filename, config_filename, log_config_file); - return server.Start(); + server.Start(); + + /* Handle Signal */ + signal(SIGHUP, server::SignalUtil::HandleSignal); + signal(SIGINT, server::SignalUtil::HandleSignal); + signal(SIGUSR1, server::SignalUtil::HandleSignal); + signal(SIGSEGV, server::SignalUtil::HandleSignal); + signal(SIGUSR2, server::SignalUtil::HandleSignal); + signal(SIGTERM, server::SignalUtil::HandleSignal); + + /* wait signal */ + pause(); + + return 0; } void print_help(const std::string &app_name) { - std::cout << std::endl<< "Usage: " << app_name << " [OPTIONS]" << std::endl << std::endl; + std::cout << std::endl << "Usage: " << app_name << " [OPTIONS]" << std::endl << std::endl; std::cout << " Options:" << std::endl; std::cout << " -h --help Print this help" << std::endl; std::cout << " -c --conf_file filename Read configuration from the file" << std::endl; diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index e4cb7b8d4a..38b41a5331 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -17,7 +17,7 @@ #include #include "Server.h" -#include "server/grpc_impl/GrpcMilvusServer.h" +#include "server/grpc_impl/GrpcServer.h" #include "utils/Log.h" #include "utils/LogUtil.h" #include "utils/SignalUtil.h" @@ -154,74 +154,61 @@ Server::Daemonize() { } } -int +void Server::Start() { - if (daemonized_) { Daemonize(); } - do { - try { - // Read config file - if (LoadConfig() != SERVER_SUCCESS) { - return 1; - } - - //log path is defined by LoadConfig, so InitLog must be called after LoadConfig - ServerConfig &config = ServerConfig::GetInstance(); - ConfigNode server_config = config.GetConfig(CONFIG_SERVER); - - std::string time_zone = server_config.GetValue(CONFIG_TIME_ZONE, "UTC+8"); - if (time_zone.length() == 3) { - time_zone = "CUT"; - } else { - int time_bias = std::stoi(time_zone.substr(3, std::string::npos)); - if (time_bias == 0) - time_zone = "CUT"; - else if (time_bias > 0) { - time_zone = "CUT" + std::to_string(-time_bias); - } else { - time_zone = "CUT+" + std::to_string(-time_bias); - } - } - - if (setenv("TZ", time_zone.c_str(), 1) != 0) { - return -1; - } - tzset(); - - InitLog(log_config_file_); - - // Handle Signal - signal(SIGINT, SignalUtil::HandleSignal); - signal(SIGHUP, SignalUtil::HandleSignal); - signal(SIGTERM, SignalUtil::HandleSignal); - server::Metrics::GetInstance().Init(); - server::SystemInfo::GetInstance().Init(); - - std::cout << "Milvus server start successfully." << std::endl; - StartService(); - - while (running_) { - sleep(10); - } - - } catch (std::exception &ex) { - std::cerr << "Milvus server encounter exception: " << std::string(ex.what()) - << "Is another server instance running?"; - break; + try { + /* Read config file */ + if (LoadConfig() != SERVER_SUCCESS) { + std::cerr << "Milvus server fail to load config file" << std::endl; + return; } - } while (false); - return 0; + /* log path is defined in Config file, so InitLog must be called after LoadConfig */ + ServerConfig &config = ServerConfig::GetInstance(); + ConfigNode server_config = config.GetConfig(CONFIG_SERVER); + + std::string time_zone = server_config.GetValue(CONFIG_TIME_ZONE, "UTC+8"); + if (time_zone.length() == 3) { + time_zone = "CUT"; + } else { + int time_bias = std::stoi(time_zone.substr(3, std::string::npos)); + if (time_bias == 0) + time_zone = "CUT"; + else if (time_bias > 0) { + time_zone = "CUT" + std::to_string(-time_bias); + } else { + time_zone = "CUT+" + std::to_string(-time_bias); + } + } + + if (setenv("TZ", time_zone.c_str(), 1) != 0) { + std::cerr << "Fail to setenv" << std::endl; + return; + } + tzset(); + + InitLog(log_config_file_); + + server::Metrics::GetInstance().Init(); + server::SystemInfo::GetInstance().Init(); + + std::cout << "Milvus server start successfully." << std::endl; + StartService(); + + } catch (std::exception &ex) { + std::cerr << "Milvus server encounter exception: " << ex.what(); + } } void Server::Stop() { std::cerr << "Milvus server is going to shutdown ..." << std::endl; - // Unlock and close lockfile + /* Unlock and close lockfile */ if (pid_fd != -1) { int ret = lockf(pid_fd, F_ULOCK, 0); if (ret != 0) { @@ -235,7 +222,7 @@ Server::Stop() { } } - // Try to delete lockfile + /* delete lockfile */ if (!pid_filename_.empty()) { int ret = unlink(pid_filename_.c_str()); if (ret != 0) { @@ -245,10 +232,6 @@ Server::Stop() { } StopService(); - { - std::lock_guard lock(mutex_); - running_ = false; - } std::cerr << "Milvus server is closed!" << std::endl; } @@ -271,12 +254,12 @@ Server::StartService() { engine::KnowhereResource::Initialize(); engine::StartSchedulerService(); DBWrapper::GetInstance().StartService(); - grpc::GrpcMilvusServer::GetInstance().Start(); + grpc::GrpcServer::GetInstance().Start(); } void Server::StopService() { - grpc::GrpcMilvusServer::GetInstance().Stop(); + grpc::GrpcServer::GetInstance().Stop(); DBWrapper::GetInstance().StopService(); engine::StopSchedulerService(); engine::KnowhereResource::Finalize(); diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index a4757cd0c6..2c912c3e91 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -37,13 +37,11 @@ class Server { const std::string &config_filename, const std::string &log_config_file); - int Start(); - + void Start(); void Stop(); private: Server(); - ~Server(); void Daemonize(); @@ -51,7 +49,6 @@ class Server { ErrorCode LoadConfig(); void StartService(); - void StopService(); private: @@ -60,9 +57,6 @@ class Server { std::string pid_filename_; std::string config_filename_; std::string log_config_file_; - - std::mutex mutex_; - bool running_ = true; }; // Server } // server diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index e49ea48236..ad0abd713d 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -23,7 +23,7 @@ #include "utils/ValidationUtil.h" #include "../DBWrapper.h" #include "version.h" -#include "GrpcMilvusServer.h" +#include "GrpcServer.h" #include "db/Utils.h" #include "scheduler/SchedInst.h" //#include diff --git a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp b/cpp/src/server/grpc_impl/GrpcServer.cpp similarity index 90% rename from cpp/src/server/grpc_impl/GrpcMilvusServer.cpp rename to cpp/src/server/grpc_impl/GrpcServer.cpp index 1af3557488..4ff7a8200f 100644 --- a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcServer.cpp @@ -16,7 +16,7 @@ // under the License. #include "milvus.grpc.pb.h" -#include "GrpcMilvusServer.h" +#include "GrpcServer.h" #include "server/ServerConfig.h" #include "server/DBWrapper.h" #include "utils/Log.h" @@ -34,7 +34,6 @@ #include #include #include -#include namespace zilliz { @@ -52,18 +51,19 @@ class NoReusePortOption : public ::grpc::ServerBuilderOption { args->SetInt(GRPC_ARG_ALLOW_REUSEPORT, 0); } - void UpdatePlugins(std::vector> * - plugins) override {} + void UpdatePlugins(std::vector> *plugins) override { + + } }; void -GrpcMilvusServer::Start() { - thread_ptr_ = std::make_shared(&GrpcMilvusServer::StartService, this); +GrpcServer::Start() { + thread_ptr_ = std::make_shared(&GrpcServer::StartService, this); } void -GrpcMilvusServer::Stop() { +GrpcServer::Stop() { StopService(); if (thread_ptr_) { thread_ptr_->join(); @@ -72,7 +72,7 @@ GrpcMilvusServer::Stop() { } Status -GrpcMilvusServer::StartService() { +GrpcServer::StartService() { ServerConfig &config = ServerConfig::GetInstance(); ConfigNode server_config = config.GetConfig(CONFIG_SERVER); ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); @@ -102,7 +102,7 @@ GrpcMilvusServer::StartService() { } Status -GrpcMilvusServer::StopService() { +GrpcServer::StopService() { if (server_ptr_ != nullptr) { server_ptr_->Shutdown(); } diff --git a/cpp/src/server/grpc_impl/GrpcMilvusServer.h b/cpp/src/server/grpc_impl/GrpcServer.h similarity index 87% rename from cpp/src/server/grpc_impl/GrpcMilvusServer.h rename to cpp/src/server/grpc_impl/GrpcServer.h index 40df193a2f..a861692fac 100644 --- a/cpp/src/server/grpc_impl/GrpcMilvusServer.h +++ b/cpp/src/server/grpc_impl/GrpcServer.h @@ -24,15 +24,16 @@ #include #include + namespace zilliz { namespace milvus { namespace server { namespace grpc { -class GrpcMilvusServer { +class GrpcServer { public: - static GrpcMilvusServer& GetInstance() { - static GrpcMilvusServer grpc_server; + static GrpcServer &GetInstance() { + static GrpcServer grpc_server; return grpc_server; } @@ -40,8 +41,8 @@ class GrpcMilvusServer { void Stop(); private: - GrpcMilvusServer() = default; - ~GrpcMilvusServer() = default; + GrpcServer() = default; + ~GrpcServer() = default; Status StartService(); Status StopService(); diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index 4da8d31eb0..8677016cf3 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -22,27 +22,28 @@ #include #include + namespace zilliz { namespace milvus { namespace server { -void SignalUtil::HandleSignal(int signum){ +void SignalUtil::HandleSignal(int signum) { - switch(signum){ + switch (signum) { case SIGINT: - case SIGUSR2:{ + case SIGUSR2: { SERVER_LOG_INFO << "Server received signal: " << signum; - server::Server& server_ptr = server::Server::Instance(); + server::Server &server_ptr = server::Server::Instance(); server_ptr.Stop(); exit(0); } - default:{ + default: { SERVER_LOG_INFO << "Server received critical signal: " << signum; SignalUtil::PrintStacktrace(); - server::Server& server_ptr = server::Server::Instance(); + server::Server &server_ptr = server::Server::Instance(); server_ptr.Stop(); exit(1); @@ -54,9 +55,9 @@ void SignalUtil::PrintStacktrace() { SERVER_LOG_INFO << "Call stack:"; const int size = 32; - void* array[size]; + void *array[size]; int stack_num = backtrace(array, size); - char ** stacktrace = backtrace_symbols(array, stack_num); + char **stacktrace = backtrace_symbols(array, stack_num); for (int i = 0; i < stack_num; ++i) { std::string info = stacktrace[i]; SERVER_LOG_INFO << info; @@ -64,7 +65,6 @@ void SignalUtil::PrintStacktrace() { free(stacktrace); } - } } } From 19d7c065acf7cf408b7f700a3885f21abd6c52cc Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 19 Sep 2019 14:46:09 +0800 Subject: [PATCH 10/16] MS-572 correct format Former-commit-id: 0bdbc27634eed7ba241b40d5957f5689d50d6a6a --- cpp/src/main.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 5819463d60..54f9541eff 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -87,13 +87,17 @@ main(int argc, char *argv[]) { break; } - case 'd':start_daemonized = 1; + case 'd': + start_daemonized = 1; break; - case 'h':print_help(app_name); + case 'h': + print_help(app_name); return EXIT_SUCCESS; - case '?':print_help(app_name); + case '?': + print_help(app_name); return EXIT_FAILURE; - default:print_help(app_name); + default: + print_help(app_name); break; } } From 02270a6298a91856926a64b274e47ccdf7dd7eb4 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Thu, 19 Sep 2019 14:52:14 +0800 Subject: [PATCH 11/16] MS-568 Fix GpuResource free error Former-commit-id: 52acd35e93a5293c70c45bb681fc54046b43a2cb --- cpp/CHANGELOG.md | 1 + cpp/src/core/knowhere/CMakeLists.txt | 1 + .../vector_index/FaissGpuResourceMgr.cpp | 128 +++++++++++++++++ .../index/vector_index/FaissGpuResourceMgr.h | 130 ++++++++++++++++++ .../knowhere/index/vector_index/gpu_ivf.cpp | 119 ++-------------- .../knowhere/index/vector_index/gpu_ivf.h | 106 +------------- .../knowhere/index/vector_index/idmap.cpp | 2 +- cpp/src/core/test/CMakeLists.txt | 2 + cpp/src/wrapper/KnowhereResource.cpp | 1 + cpp/src/wrapper/KnowhereResource.h | 1 - 10 files changed, 276 insertions(+), 215 deletions(-) create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.h diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 9af19afe26..63a1e887b6 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -5,6 +5,7 @@ Please mark all change in change log and use the ticket from JIRA. # Milvus 0.5.0 (TODO) ## Bug +- MS-568 - Fix gpuresource free error ## Improvement - MS-552 - Add and change the easylogging library diff --git a/cpp/src/core/knowhere/CMakeLists.txt b/cpp/src/core/knowhere/CMakeLists.txt index 24cae1d6f9..55f23e1095 100644 --- a/cpp/src/core/knowhere/CMakeLists.txt +++ b/cpp/src/core/knowhere/CMakeLists.txt @@ -48,6 +48,7 @@ set(index_srcs knowhere/index/vector_index/nsg/nsg_io.cpp knowhere/index/vector_index/nsg/utils.cpp knowhere/index/vector_index/cloner.cpp + knowhere/index/vector_index/FaissGpuResourceMgr.cpp ) set(depend_libs diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp new file mode 100644 index 0000000000..c50594b4b0 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include "FaissGpuResourceMgr.h" + + +namespace zilliz { +namespace knowhere { + +FaissGpuResourceMgr &FaissGpuResourceMgr::GetInstance() { + static FaissGpuResourceMgr instance; + return instance; +} + +void FaissGpuResourceMgr::AllocateTempMem(ResPtr &resource, + const int64_t &device_id, + const int64_t &size) { + if (size) { + resource->faiss_res->setTempMemory(size); + } + else { + auto search = devices_params_.find(device_id); + if (search != devices_params_.end()) { + resource->faiss_res->setTempMemory(search->second.temp_mem_size); + } + // else do nothing. allocate when use. + } +} + +void FaissGpuResourceMgr::InitDevice(int64_t device_id, + int64_t pin_mem_size, + int64_t temp_mem_size, + int64_t res_num) { + DeviceParams params; + params.pinned_mem_size = pin_mem_size; + params.temp_mem_size = temp_mem_size; + params.resource_num = res_num; + + devices_params_.emplace(device_id, params); +} + +void FaissGpuResourceMgr::InitResource() { + if(is_init) return ; + + is_init = true; + + //std::cout << "InitResource" << std::endl; + for(auto& device : devices_params_) { + auto& device_id = device.first; + + mutex_cache_.emplace(device_id, std::make_unique()); + + //std::cout << "Device Id: " << device_id << std::endl; + auto& device_param = device.second; + auto& bq = idle_map_[device_id]; + + for (int64_t i = 0; i < device_param.resource_num; ++i) { + //std::cout << "Resource Id: " << i << std::endl; + auto raw_resource = std::make_shared(); + + // TODO(linxj): enable set pinned memory + auto res_wrapper = std::make_shared(raw_resource); + AllocateTempMem(res_wrapper, device_id, 0); + + bq.Put(res_wrapper); + } + } + //std::cout << "End initResource" << std::endl; +} + +ResPtr FaissGpuResourceMgr::GetRes(const int64_t &device_id, + const int64_t &alloc_size) { + InitResource(); + + auto finder = idle_map_.find(device_id); + if (finder != idle_map_.end()) { + auto& bq = finder->second; + auto&& resource = bq.Take(); + AllocateTempMem(resource, device_id, alloc_size); + return resource; + } + return nullptr; +} + +void FaissGpuResourceMgr::MoveToIdle(const int64_t &device_id, const ResPtr &res) { + auto finder = idle_map_.find(device_id); + if (finder != idle_map_.end()) { + auto& bq = finder->second; + bq.Put(res); + } +} + +void FaissGpuResourceMgr::Free() { + for (auto &item : idle_map_) { + auto& bq = item.second; + while (!bq.Empty()) { + bq.Take(); + } + } + is_init = false; +} + +void +FaissGpuResourceMgr::Dump() { + for (auto &item : idle_map_) { + auto& bq = item.second; + std::cout << "device_id: " << item.first + << ", resource count:" << bq.Size(); + } +} + +} // knowhere +} // zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.h b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.h new file mode 100644 index 0000000000..3b4786a2a5 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.h @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include +#include +#include + +#include + +#include "src/utils/BlockingQueue.h" + +namespace zilliz { +namespace knowhere { + +struct Resource { + explicit Resource(std::shared_ptr &r) : faiss_res(r) { + static int64_t global_id = 0; + id = global_id++; + } + + std::shared_ptr faiss_res; + int64_t id; + std::mutex mutex; +}; +using ResPtr = std::shared_ptr; +using ResWPtr = std::weak_ptr; + +class FaissGpuResourceMgr { +public: + friend class ResScope; + using ResBQ = zilliz::milvus::server::BlockingQueue; + +public: + struct DeviceParams { + int64_t temp_mem_size = 0; + int64_t pinned_mem_size = 0; + int64_t resource_num = 2; + }; + +public: + static FaissGpuResourceMgr & + GetInstance(); + + // Free gpu resource, avoid cudaGetDevice error when deallocate. + // this func should be invoke before main return + void + Free(); + + void + AllocateTempMem(ResPtr &resource, const int64_t& device_id, const int64_t& size); + + void + InitDevice(int64_t device_id, + int64_t pin_mem_size = 0, + int64_t temp_mem_size = 0, + int64_t res_num = 2); + + void + InitResource(); + + // allocate gpu memory invoke by build or copy_to_gpu + ResPtr + GetRes(const int64_t &device_id, const int64_t& alloc_size = 0); + + void + MoveToIdle(const int64_t &device_id, const ResPtr& res); + + void + Dump(); + +protected: + bool is_init = false; + + std::map> mutex_cache_; + std::map devices_params_; + std::map idle_map_; +}; + +class ResScope { +public: + ResScope(ResPtr &res, const int64_t& device_id, const bool& isown) + : resource(res), device_id(device_id), move(true), own(isown) { + Lock(); + } + + // specif for search + // get the ownership of gpuresource and gpu + ResScope(ResWPtr &res, const int64_t &device_id) + :device_id(device_id),move(false),own(true) { + resource = res.lock(); + Lock(); + } + + void Lock() { + if (own) FaissGpuResourceMgr::GetInstance().mutex_cache_[device_id]->lock(); + resource->mutex.lock(); + } + + ~ResScope() { + if (own) FaissGpuResourceMgr::GetInstance().mutex_cache_[device_id]->unlock(); + if (move) FaissGpuResourceMgr::GetInstance().MoveToIdle(device_id, resource); + resource->mutex.unlock(); + } + +private: + ResPtr resource; // hold resource until deconstruct + int64_t device_id; + bool move = true; + bool own = false; +}; + +} // knowhere +} // zilliz \ No newline at end of file diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp index fd04501c83..4efd5f24ae 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp @@ -67,9 +67,9 @@ void GPUIVF::set_index_model(IndexModelPtr model) { auto host_index = std::static_pointer_cast(model); if (auto gpures = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_)) { ResScope rs(gpures, gpu_id_, false); - res_ = gpures; - auto device_index = faiss::gpu::index_cpu_to_gpu(res_->faiss_res.get(), gpu_id_, host_index->index_.get()); + auto device_index = faiss::gpu::index_cpu_to_gpu(gpures->faiss_res.get(), gpu_id_, host_index->index_.get()); index_.reset(device_index); + res_ = gpures; } else { KNOWHERE_THROW_MSG("load index model error, can't get gpu_resource"); } @@ -114,9 +114,9 @@ void GPUIVF::LoadImpl(const BinarySet &index_binary) { if (auto temp_res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_)) { ResScope rs(temp_res, gpu_id_, false); - res_ = temp_res; - auto device_index = faiss::gpu::index_cpu_to_gpu(res_->faiss_res.get(), gpu_id_, index); + auto device_index = faiss::gpu::index_cpu_to_gpu(temp_res->faiss_res.get(), gpu_id_, index); index_.reset(device_index); + res_ = temp_res; } else { KNOWHERE_THROW_MSG("Load error, can't get gpu resource"); } @@ -176,12 +176,13 @@ VectorIndexPtr GPUIVF::CopyGpuToGpu(const int64_t &device_id, const Config &conf auto host_index = CopyGpuToCpu(config); return std::static_pointer_cast(host_index)->CopyCpuToGpu(device_id, config); } + void GPUIVF::Add(const DatasetPtr &dataset, const Config &config) { - auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); - if (temp_resource != nullptr) { - ResScope rs(temp_resource, gpu_id_, true); + if (auto spt = res_.lock()) { + ResScope rs(res_, gpu_id_); IVF::Add(dataset, config); - } else { + } + else { KNOWHERE_THROW_MSG("Add IVF can't get gpu resource"); } } @@ -264,108 +265,6 @@ VectorIndexPtr GPUIVFSQ::CopyGpuToCpu(const Config &config) { return std::make_shared(new_index); } -FaissGpuResourceMgr &FaissGpuResourceMgr::GetInstance() { - static FaissGpuResourceMgr instance; - return instance; -} - -void FaissGpuResourceMgr::AllocateTempMem(ResPtr &resource, - const int64_t &device_id, - const int64_t &size) { - if (size) { - resource->faiss_res->setTempMemory(size); - } - else { - auto search = devices_params_.find(device_id); - if (search != devices_params_.end()) { - resource->faiss_res->setTempMemory(search->second.temp_mem_size); - } - // else do nothing. allocate when use. - } -} - -void FaissGpuResourceMgr::InitDevice(int64_t device_id, - int64_t pin_mem_size, - int64_t temp_mem_size, - int64_t res_num) { - DeviceParams params; - params.pinned_mem_size = pin_mem_size; - params.temp_mem_size = temp_mem_size; - params.resource_num = res_num; - - devices_params_.emplace(device_id, params); -} - -void FaissGpuResourceMgr::InitResource() { - if(is_init) return ; - - is_init = true; - - //std::cout << "InitResource" << std::endl; - for(auto& device : devices_params_) { - auto& device_id = device.first; - - mutex_cache_.emplace(device_id, std::make_unique()); - - //std::cout << "Device Id: " << device_id << std::endl; - auto& device_param = device.second; - auto& bq = idle_map_[device_id]; - - for (int64_t i = 0; i < device_param.resource_num; ++i) { - //std::cout << "Resource Id: " << i << std::endl; - auto raw_resource = std::make_shared(); - - // TODO(linxj): enable set pinned memory - auto res_wrapper = std::make_shared(raw_resource); - AllocateTempMem(res_wrapper, device_id, 0); - - bq.Put(res_wrapper); - } - } - //std::cout << "End initResource" << std::endl; -} - -ResPtr FaissGpuResourceMgr::GetRes(const int64_t &device_id, - const int64_t &alloc_size) { - InitResource(); - - auto finder = idle_map_.find(device_id); - if (finder != idle_map_.end()) { - auto& bq = finder->second; - auto&& resource = bq.Take(); - AllocateTempMem(resource, device_id, alloc_size); - return resource; - } - return nullptr; -} - -void FaissGpuResourceMgr::MoveToIdle(const int64_t &device_id, const ResPtr &res) { - auto finder = idle_map_.find(device_id); - if (finder != idle_map_.end()) { - auto& bq = finder->second; - bq.Put(res); - } -} - -void FaissGpuResourceMgr::Free() { - for (auto &item : idle_map_) { - auto& bq = item.second; - while (!bq.Empty()) { - bq.Take(); - } - } - is_init = false; -} - -void -FaissGpuResourceMgr::Dump() { - for (auto &item : idle_map_) { - auto& bq = item.second; - std::cout << "device_id: " << item.first - << ", resource count:" << bq.Size(); - } -} - void GPUIndex::SetGpuDevice(const int &gpu_id) { gpu_id_ = gpu_id; } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h index 429a94167a..4948d16ac4 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h @@ -18,118 +18,18 @@ #pragma once -#include #include "ivf.h" -#include "src/utils/BlockingQueue.h" +#include "FaissGpuResourceMgr.h" namespace zilliz { namespace knowhere { -struct Resource { - explicit Resource(std::shared_ptr &r): faiss_res(r) { - static int64_t global_id = 0; - id = global_id++; - } - - std::shared_ptr faiss_res; - int64_t id; - std::mutex mutex; -}; -using ResPtr = std::shared_ptr; -using ResWPtr = std::weak_ptr; - -class FaissGpuResourceMgr { - public: - friend class ResScope; - - public: - using ResBQ = zilliz::milvus::server::BlockingQueue; - - struct DeviceParams { - int64_t temp_mem_size = 0; - int64_t pinned_mem_size = 0; - int64_t resource_num = 2; - }; - - public: - static FaissGpuResourceMgr & - GetInstance(); - - // Free gpu resource, avoid cudaGetDevice error when deallocate. - // this func should be invoke before main return - void - Free(); - - void - AllocateTempMem(ResPtr &resource, const int64_t& device_id, const int64_t& size); - - void - InitDevice(int64_t device_id, - int64_t pin_mem_size = 0, - int64_t temp_mem_size = 0, - int64_t res_num = 2); - - void - InitResource(); - - // allocate gpu memory invoke by build or copy_to_gpu - ResPtr - GetRes(const int64_t &device_id, const int64_t& alloc_size = 0); - - // allocate gpu memory before search - // this func will return True if the device is idle and exists an idle resource. - //bool - //GetRes(const int64_t& device_id, ResPtr &res, const int64_t& alloc_size = 0); - - void - MoveToIdle(const int64_t &device_id, const ResPtr& res); - - void - Dump(); - - protected: - bool is_init = false; - - std::map> mutex_cache_; - std::map devices_params_; - std::map idle_map_; -}; - -class ResScope { - public: - ResScope(ResPtr &res, const int64_t& device_id, const bool& isown) - : resource(res), device_id(device_id), move(true), own(isown) { - if (isown) FaissGpuResourceMgr::GetInstance().mutex_cache_[device_id]->lock(); - res->mutex.lock(); - } - - // specif for search - // get the ownership of gpuresource and gpu - ResScope(ResPtr &res, const int64_t &device_id) - : resource(res), device_id(device_id), move(false), own(true) { - FaissGpuResourceMgr::GetInstance().mutex_cache_[device_id]->lock(); - res->mutex.lock(); - } - - ~ResScope() { - if (own) FaissGpuResourceMgr::GetInstance().mutex_cache_[device_id]->unlock(); - if (move) FaissGpuResourceMgr::GetInstance().MoveToIdle(device_id, resource); - resource->mutex.unlock(); - } - - private: - ResPtr resource; - int64_t device_id; - bool move = true; - bool own = false; -}; - class GPUIndex { public: explicit GPUIndex(const int &device_id) : gpu_id_(device_id) {} - GPUIndex(const int& device_id, ResPtr resource): gpu_id_(device_id), res_(std::move(resource)){} + GPUIndex(const int& device_id, const ResPtr& resource): gpu_id_(device_id), res_(resource){} virtual VectorIndexPtr CopyGpuToCpu(const Config &config) = 0; virtual VectorIndexPtr CopyGpuToGpu(const int64_t &device_id, const Config &config) = 0; @@ -139,7 +39,7 @@ class GPUIndex { protected: int64_t gpu_id_; - ResPtr res_ = nullptr; + ResWPtr res_; }; class GPUIVF : public IVF, public GPUIndex { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp index b4085e6df8..494da4438a 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp @@ -224,9 +224,9 @@ void GPUIDMAP::LoadImpl(const BinarySet &index_binary) { if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_) ){ ResScope rs(res, gpu_id_, false); - res_ = res; auto device_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), gpu_id_, index); index_.reset(device_index); + res_ = res; } else { KNOWHERE_THROW_MSG("Load error, can't get gpu resource"); } diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index 5b8dca2e4b..b62fd6cda3 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -32,6 +32,7 @@ set(ivf_srcs ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp utils.cpp ) if(NOT TARGET test_ivf) @@ -48,6 +49,7 @@ set(idmap_srcs ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp utils.cpp ) if(NOT TARGET test_idmap) diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index afbeb81816..c8c03423a8 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -17,6 +17,7 @@ #include "KnowhereResource.h" +#include "knowhere/index/vector_index/FaissGpuResourceMgr.h" #include "server/ServerConfig.h" #include diff --git a/cpp/src/wrapper/KnowhereResource.h b/cpp/src/wrapper/KnowhereResource.h index c14234eaed..5a7b9aa6c5 100644 --- a/cpp/src/wrapper/KnowhereResource.h +++ b/cpp/src/wrapper/KnowhereResource.h @@ -19,7 +19,6 @@ #pragma once #include "utils/Error.h" -#include "knowhere/index/vector_index/gpu_ivf.h" namespace zilliz { namespace milvus { From dc71feababde36c539e9a55646d919978fffa54e Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 19 Sep 2019 14:52:25 +0800 Subject: [PATCH 12/16] use better name Former-commit-id: ccf466b6906067d03cb62f0a813a3d7b4c346e2c --- cpp/src/server/Server.h | 1 - cpp/src/utils/SignalUtil.cpp | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index 2c912c3e91..30ea95a7bb 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -21,7 +21,6 @@ #include #include -#include namespace zilliz { diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index 8677016cf3..6c68f702d2 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -34,8 +34,8 @@ void SignalUtil::HandleSignal(int signum) { case SIGUSR2: { SERVER_LOG_INFO << "Server received signal: " << signum; - server::Server &server_ptr = server::Server::Instance(); - server_ptr.Stop(); + server::Server &server = server::Server::Instance(); + server.Stop(); exit(0); } @@ -43,8 +43,8 @@ void SignalUtil::HandleSignal(int signum) { SERVER_LOG_INFO << "Server received critical signal: " << signum; SignalUtil::PrintStacktrace(); - server::Server &server_ptr = server::Server::Instance(); - server_ptr.Stop(); + server::Server &server = server::Server::Instance(); + server.Stop(); exit(1); } From 029d4a97b3967e2ac6f150562c7f2a15bca7fec6 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Thu, 19 Sep 2019 15:12:46 +0800 Subject: [PATCH 13/16] MS-573 Enable log in knowhere Former-commit-id: 7b60e5fc6edcaabea9e967ba4f3312a17533c986 --- cpp/src/core/test/CMakeLists.txt | 9 ++++++--- cpp/src/core/test/test_nsg/CMakeLists.txt | 1 + cpp/src/core/test/utils.cpp | 9 +++++++++ cpp/src/core/test/utils.h | 3 +++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index 5b8dca2e4b..d70007229b 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -22,6 +22,9 @@ set(basic_libs gomp gfortran pthread ) +set(util_srcs + ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc + ) # set(ivf_srcs @@ -35,7 +38,7 @@ set(ivf_srcs utils.cpp ) if(NOT TARGET test_ivf) - add_executable(test_ivf test_ivf.cpp ${ivf_srcs}) + add_executable(test_ivf test_ivf.cpp ${ivf_srcs} ${util_srcs}) endif() target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) @@ -51,7 +54,7 @@ set(idmap_srcs utils.cpp ) if(NOT TARGET test_idmap) - add_executable(test_idmap test_idmap.cpp ${idmap_srcs}) + add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${util_srcs}) endif() target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) @@ -68,7 +71,7 @@ set(kdt_srcs utils.cpp ) if(NOT TARGET test_kdt) - add_executable(test_kdt test_kdt.cpp ${kdt_srcs}) + add_executable(test_kdt test_kdt.cpp ${kdt_srcs} ${util_srcs}) endif() target_link_libraries(test_kdt SPTAGLibStatic diff --git a/cpp/src/core/test/test_nsg/CMakeLists.txt b/cpp/src/core/test/test_nsg/CMakeLists.txt index 9ee8e76525..17b62cd40d 100644 --- a/cpp/src/core/test/test_nsg/CMakeLists.txt +++ b/cpp/src/core/test/test_nsg/CMakeLists.txt @@ -34,6 +34,7 @@ if(NOT TARGET test_nsg) test_nsg.cpp ${interface_src} ${nsg_src} + ${util_srcs} ) endif() diff --git a/cpp/src/core/test/utils.cpp b/cpp/src/core/test/utils.cpp index 97f4600ca7..4e240f5f74 100644 --- a/cpp/src/core/test/utils.cpp +++ b/cpp/src/core/test/utils.cpp @@ -18,6 +18,15 @@ #include "utils.h" +INITIALIZE_EASYLOGGINGPP + +void InitLog() { + el::Configurations defaultConf; + defaultConf.setToDefault(); + defaultConf.set(el::Level::Debug, + el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)"); + el::Loggers::reconfigureLogger("default", defaultConf); +} void DataGen::Init_with_default() { Generate(dim, nb, nq); diff --git a/cpp/src/core/test/utils.h b/cpp/src/core/test/utils.h index b9db01de51..df7f11d689 100644 --- a/cpp/src/core/test/utils.h +++ b/cpp/src/core/test/utils.h @@ -24,6 +24,7 @@ #include #include "knowhere/adapter/structure.h" +#include "knowhere/common/Log.h" class DataGen { protected: @@ -65,6 +66,8 @@ extern void GenBase(const int64_t &dim, float *xb, int64_t *ids); +extern void InitLog(); + zilliz::knowhere::DatasetPtr generate_dataset(int64_t nb, int64_t dim, float *xb, long *ids); From 6ed8251c3b3da41782a67989d9dafd4b570937ef Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 19 Sep 2019 15:15:35 +0800 Subject: [PATCH 14/16] fix unitest failed issue Former-commit-id: b4f12f185be2a302c25fa04695966f49c2a446ed --- cpp/unittest/db/mem_test.cpp | 59 +++++++++++++++++++----------------- cpp/unittest/db/utils.cpp | 8 +++-- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 4e958795ad..b85718a865 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -63,8 +63,8 @@ void BuildVectors(int64_t n, std::vector &vectors) { vectors.resize(n * TABLE_DIM); float *data = vectors.data(); for (int i = 0; i < n; i++) { - for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); - data[TABLE_DIM * i] += i / 2000.; + for (int j = 0; j < TABLE_DIM; j++) + data[TABLE_DIM * i + j] = drand48(); } } } @@ -224,39 +224,42 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - std::map> search_vectors; - { - engine::IDNumbers vector_ids; - int64_t nb = 100000; - std::vector xb; - BuildVectors(nb, xb); - stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - ASSERT_TRUE(stat.ok()); + int64_t nb = 100000; + std::vector xb; + BuildVectors(nb, xb); - std::this_thread::sleep_for(std::chrono::seconds(3)); - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_int_distribution dis(0, nb - 1); - - int64_t num_query = 10; - for (int64_t i = 0; i < num_query; ++i) { - int64_t index = dis(gen); - std::vector search; - for (int64_t j = 0; j < TABLE_DIM; j++) { - search.push_back(xb[index * TABLE_DIM + j]); - } - search_vectors.insert(std::make_pair(vector_ids[index], search)); - } + engine::IDNumbers vector_ids; + for(int64_t i = 0; i < nb; i++) { + vector_ids.push_back(i); } - int k = 10; + stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + ASSERT_TRUE(stat.ok()); + + std::this_thread::sleep_for(std::chrono::seconds(3));//ensure raw data write to disk + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dis(0, nb - 1); + + int64_t num_query = 10; + std::map> search_vectors; + for (int64_t i = 0; i < num_query; ++i) { + int64_t index = dis(gen); + std::vector search; + for (int64_t j = 0; j < TABLE_DIM; j++) { + search.push_back(xb[index * TABLE_DIM + j]); + } + search_vectors.insert(std::make_pair(vector_ids[index], search)); + } + + int topk = 10, nprobe = 10; for (auto &pair : search_vectors) { auto &search = pair.second; engine::QueryResults results; - stat = db_->Query(TABLE_NAME, k, 1, 10, search.data(), results); + stat = db_->Query(TABLE_NAME, topk, 1, nprobe, search.data(), results); ASSERT_EQ(results[0][0].first, pair.first); - ASSERT_LT(results[0][0].second, 0.00001); + ASSERT_LT(results[0][0].second, 1e-4); } } diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 982e98abd3..03ea11bf08 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -21,6 +21,8 @@ #include #include "utils.h" +#include "cache/GpuCacheMgr.h" +#include "cache/CpuCacheMgr.h" #include "db/DBFactory.h" #include "db/Options.h" #include "server/ServerConfig.h" @@ -63,6 +65,8 @@ void BaseTest::SetUp() { } void BaseTest::TearDown() { + zilliz::milvus::cache::CpuCacheMgr::GetInstance()->ClearCache(); + zilliz::milvus::cache::GpuCacheMgr::GetInstance(0)->ClearCache(); zilliz::knowhere::FaissGpuResourceMgr::GetInstance().Free(); } @@ -100,12 +104,12 @@ void DBTest::TearDown() { db_->Stop(); db_->DropAll(); - BaseTest::TearDown(); - engine::JobMgrInst::GetInstance()->Stop(); engine::SchedInst::GetInstance()->Stop(); engine::ResMgrInst::GetInstance()->Stop(); + BaseTest::TearDown(); + auto options = GetOptions(); boost::filesystem::remove_all(options.meta.path); } From e26948e85859e7f5b327b7a238c20f92dcc2c421 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Thu, 19 Sep 2019 15:18:37 +0800 Subject: [PATCH 15/16] MS-573 remove useless code Former-commit-id: 4db9733a33dcbc26b5b11a66f54087d3af51ccc6 --- cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp index 51fe237658..fd04501c83 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp @@ -30,7 +30,6 @@ #include "cloner.h" #include "knowhere/adapter/faiss_adopt.h" #include "gpu_ivf.h" -#include "knowhere/common/Log.h" #include @@ -224,7 +223,6 @@ VectorIndexPtr GPUIVFPQ::CopyGpuToCpu(const Config &config) { } IndexModelPtr GPUIVFSQ::Train(const DatasetPtr &dataset, const Config &config) { - KNOWHERE_LOG_WARNING << "lalalalalalala"; auto nlist = config["nlist"].as(); auto nbits = config["nbits"].as(); // TODO(linxj): gpu only support SQ4 SQ8 SQ16 gpu_id_ = config.get_with_default("gpu_id", gpu_id_); From c72d8ac14b50d14c59825361382eea0ab42b628e Mon Sep 17 00:00:00 2001 From: test Date: Thu, 19 Sep 2019 08:25:12 +0000 Subject: [PATCH 16/16] MS-575 Add Clang-format & Clang-tidy & Cpplint Former-commit-id: 226d8cc0deeb9062227da6f36d3d684d80ca4d18 --- cpp/CHANGELOG.md | 1 + cpp/CMakeLists.txt | 14 ++- cpp/build-support/clang_format_exclusions.txt | 0 cpp/build-support/run_clang_format.py | 79 +++++++++++++ cpp/cmake/FindClangTools.cmake | 104 ++++++++++++++++++ 5 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 cpp/build-support/clang_format_exclusions.txt create mode 100755 cpp/build-support/run_clang_format.py create mode 100644 cpp/cmake/FindClangTools.cmake diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 820b16756b..60374fa310 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -25,6 +25,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-561 - Add contributing guidelines, code of conduct and README docs - MS-567 - Add NOTICE.md - MS-569 - Complete the NOTICE.md +- MS-575 - Add Clang-format & Clang-tidy & Cpplint # Milvus 0.4.0 (2019-09-12) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index bb15ddf075..01007bf085 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -21,6 +21,8 @@ cmake_minimum_required(VERSION 3.14) message(STATUS "Building using CMake version: ${CMAKE_VERSION}") +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + MACRO (GET_CURRENT_TIME CURRENT_TIME) execute_process(COMMAND "date" +"%Y-%m-%d %H:%M.%S" OUTPUT_VARIABLE ${CURRENT_TIME}) ENDMACRO (GET_CURRENT_TIME) @@ -41,6 +43,10 @@ endif() set(MILVUS_VERSION "${GIT_BRANCH_NAME}") string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}") +set(CLANG_FORMAT_VERSION "6.0") +find_package(ClangTools) +set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") + if(CMAKE_BUILD_TYPE STREQUAL "Release") set(BUILD_TYPE "Release") else() @@ -111,8 +117,6 @@ else() include_directories(${MYSQL_INCLUDE_DIR}) endif() -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") - set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR}) set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR}) set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) @@ -152,3 +156,9 @@ install(FILES conf/log_config.conf DESTINATION conf) + +add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py + ${CLANG_FORMAT_BIN} + ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt + ${CMAKE_CURRENT_SOURCE_DIR}/src) + diff --git a/cpp/build-support/clang_format_exclusions.txt b/cpp/build-support/clang_format_exclusions.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cpp/build-support/run_clang_format.py b/cpp/build-support/run_clang_format.py new file mode 100755 index 0000000000..4235b2d4af --- /dev/null +++ b/cpp/build-support/run_clang_format.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import fnmatch +import os +import subprocess +import sys + +if len(sys.argv) < 4: + sys.stderr.write("Usage: %s $CLANG_FORMAT_VERSION exclude_globs.txt " + "$source_dir\n" % + sys.argv[0]) + sys.exit(1) + +CLANG_FORMAT = sys.argv[1] +EXCLUDE_GLOBS_FILENAME = sys.argv[2] +SOURCE_DIR = sys.argv[3] + +if len(sys.argv) > 4: + CHECK_FORMAT = int(sys.argv[4]) == 1 +else: + CHECK_FORMAT = False + + +exclude_globs = [line.strip() for line in open(EXCLUDE_GLOBS_FILENAME, "r")] + +files_to_format = [] +matches = [] +for directory, subdirs, files in os.walk(SOURCE_DIR): + for name in files: + name = os.path.join(directory, name) + if not (name.endswith('.h') or name.endswith('.cpp') or name.endswith('.cuh') or name.endswith('.cu')): + continue + + excluded = False + for g in exclude_globs: + if fnmatch.fnmatch(name, g): + excluded = True + break + if not excluded: + files_to_format.append(name) + +if CHECK_FORMAT: + output = subprocess.check_output([CLANG_FORMAT, '-output-replacements-xml'] + + files_to_format, + stderr=subprocess.STDOUT).decode('utf8') + + to_fix = [] + for line in output.split('\n'): + if 'offset' in line: + to_fix.append(line) + + if len(to_fix) > 0: + print("clang-format checks failed, run 'make format' to fix") + sys.exit(-1) +else: + try: + cmd = [CLANG_FORMAT, '-i'] + files_to_format + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except Exception as e: + print(e) + print(' '.join(cmd)) + raise + diff --git a/cpp/cmake/FindClangTools.cmake b/cpp/cmake/FindClangTools.cmake new file mode 100644 index 0000000000..075d9381c5 --- /dev/null +++ b/cpp/cmake/FindClangTools.cmake @@ -0,0 +1,104 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Tries to find the clang-tidy and clang-format modules +# +# Usage of this module as follows: +# +# find_package(ClangTools) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# ClangToolsBin_HOME - +# When set, this path is inspected instead of standard library binary locations +# to find clang-tidy and clang-format +# +# This module defines +# CLANG_TIDY_BIN, The path to the clang tidy binary +# CLANG_TIDY_FOUND, Whether clang tidy was found +# CLANG_FORMAT_BIN, The path to the clang format binary +# CLANG_TIDY_FOUND, Whether clang format was found + +find_program(CLANG_TIDY_BIN + NAMES + clang-tidy-6.0 + clang-tidy-5.0 + clang-tidy-4.0 + clang-tidy-3.9 + clang-tidy-3.8 + clang-tidy-3.7 + clang-tidy-3.6 + clang-tidy + PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin + NO_DEFAULT_PATH +) + +if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" ) + set(CLANG_TIDY_FOUND 0) + message("clang-tidy not found") +else() + set(CLANG_TIDY_FOUND 1) + message("clang-tidy found at ${CLANG_TIDY_BIN}") +endif() + +if (CLANG_FORMAT_VERSION) + find_program(CLANG_FORMAT_BIN + NAMES clang-format-${CLANG_FORMAT_VERSION} + PATHS + ${ClangTools_PATH} + $ENV{CLANG_TOOLS_PATH} + /usr/local/bin /usr/bin + NO_DEFAULT_PATH + ) + + # If not found yet, search alternative locations + if (("${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND") AND APPLE) + # Homebrew ships older LLVM versions in /usr/local/opt/llvm@version/ + STRING(REGEX REPLACE "^([0-9]+)\\.[0-9]+" "\\1" CLANG_FORMAT_MAJOR_VERSION "${CLANG_FORMAT_VERSION}") + STRING(REGEX REPLACE "^[0-9]+\\.([0-9]+)" "\\1" CLANG_FORMAT_MINOR_VERSION "${CLANG_FORMAT_VERSION}") + if ("${CLANG_FORMAT_MINOR_VERSION}" STREQUAL "0") + find_program(CLANG_FORMAT_BIN + NAMES clang-format + PATHS /usr/local/opt/llvm@${CLANG_FORMAT_MAJOR_VERSION}/bin + NO_DEFAULT_PATH + ) + else() + find_program(CLANG_FORMAT_BIN + NAMES clang-format + PATHS /usr/local/opt/llvm@${CLANG_FORMAT_VERSION}/bin + NO_DEFAULT_PATH + ) + endif() + endif() +else() + find_program(CLANG_FORMAT_BIN + NAMES clang-format-4.0 + clang-format-3.9 + clang-format-3.8 + clang-format-3.7 + clang-format-3.6 + clang-format + PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin + NO_DEFAULT_PATH + ) +endif() + +if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" ) + set(CLANG_FORMAT_FOUND 0) + message("clang-format not found") +else() + set(CLANG_FORMAT_FOUND 1) + message("clang-format found at ${CLANG_FORMAT_BIN}") +endif() +