From 41d7500f61ff8139fa82d7cf5fccc56f9bceb368 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Sat, 14 Dec 2019 15:11:02 +0800 Subject: [PATCH 1/3] support config cli test 2 (#764) * #665 support get/set config via CLI * #665 support get/set config via CLI * #665 add unittest for config CLI * #665 remove config_node_map_ * #665 remove config_node_map_ * #665 fix clang-format * #665 fix clang-format * #665 update changelog * #665 code clean * #665 update API interface * #665 update API interface * #665 handle server status * #665 handle unknown command * #665 update debug log * #665 rollback CmdRequest.cpp --- CHANGELOG.md | 1 + core/src/sdk/grpc/ClientProxy.cpp | 21 ++- core/src/sdk/grpc/ClientProxy.h | 6 + core/src/sdk/include/MilvusApi.h | 26 ++++ core/src/sdk/interface/ConnectionImpl.cpp | 9 ++ core/src/sdk/interface/ConnectionImpl.h | 6 + core/src/server/Config.cpp | 102 ++++++++++++++- core/src/server/Config.h | 12 +- core/unittest/server/test_config.cpp | 150 ++++++++++++++++++++++ 9 files changed, 329 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ca5b23622..8bc28d0c62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Please mark all change in change log and use the issue from GitHub ## Feature - \#343 - Add Opentracing +- \#665 - Support get/set config via CLI ## Improvement diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index c6a087cfe4..afa90c955b 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -334,7 +334,7 @@ ClientProxy::ServerStatus() const { try { std::string dummy; - Status status = client_ptr_->Cmd(dummy, ""); + Status status = client_ptr_->Cmd(dummy, "status"); return "server alive"; } catch (std::exception& ex) { return "connection lost"; @@ -456,4 +456,23 @@ ClientProxy::DropPartition(const PartitionParam& partition_param) { } } +Status +ClientProxy::GetConfig(const std::string& node_name, std::string& value) const { + try { + return client_ptr_->Cmd(value, "get " + node_name); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Fail to get config: " + node_name); + } +} + +Status +ClientProxy::SetConfig(const std::string& node_name, const std::string& value) const { + try { + std::string dummy; + return client_ptr_->Cmd(dummy, "set " + node_name + " " + value); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Fail to set config: " + node_name); + } +} + } // namespace milvus diff --git a/core/src/sdk/grpc/ClientProxy.h b/core/src/sdk/grpc/ClientProxy.h index 0a0ea0b8eb..c19bd3a12b 100644 --- a/core/src/sdk/grpc/ClientProxy.h +++ b/core/src/sdk/grpc/ClientProxy.h @@ -104,6 +104,12 @@ class ClientProxy : public Connection { Status DropPartition(const PartitionParam& partition_param) override; + Status + GetConfig(const std::string& node_name, std::string& value) const override; + + Status + SetConfig(const std::string& node_name, const std::string& value) const override; + private: std::shared_ptr<::grpc::Channel> channel_; diff --git a/core/src/sdk/include/MilvusApi.h b/core/src/sdk/include/MilvusApi.h index 4abdf31357..61966695d1 100644 --- a/core/src/sdk/include/MilvusApi.h +++ b/core/src/sdk/include/MilvusApi.h @@ -444,6 +444,32 @@ class Connection { */ virtual Status DropPartition(const PartitionParam& param) = 0; + + /** + * @brief Get config method + * + * This method is used to set config. + * + * @param node_name, config node name. + * @param value, config value. + * + * @return Indicate if this operation is successful. + */ + virtual Status + GetConfig(const std::string& node_name, std::string& value) const = 0; + + /** + * @brief Set config method + * + * This method is used to set config. + * + * @param node_name, config node name. + * @param value, config value. + * + * @return Indicate if this operation is successful. + */ + virtual Status + SetConfig(const std::string& node_name, const std::string& value) const = 0; }; } // namespace milvus diff --git a/core/src/sdk/interface/ConnectionImpl.cpp b/core/src/sdk/interface/ConnectionImpl.cpp index 3b544c00c0..698c2c4ea8 100644 --- a/core/src/sdk/interface/ConnectionImpl.cpp +++ b/core/src/sdk/interface/ConnectionImpl.cpp @@ -161,4 +161,13 @@ ConnectionImpl::DropPartition(const PartitionParam& param) { return client_proxy_->DropPartition(param); } +Status +ConnectionImpl::GetConfig(const std::string& node_name, std::string& value) const { + return client_proxy_->GetConfig(node_name, value); +} + +Status +ConnectionImpl::SetConfig(const std::string& node_name, const std::string& value) const { + return client_proxy_->SetConfig(node_name, value); +} } // namespace milvus diff --git a/core/src/sdk/interface/ConnectionImpl.h b/core/src/sdk/interface/ConnectionImpl.h index e22f8d33d8..93eda04662 100644 --- a/core/src/sdk/interface/ConnectionImpl.h +++ b/core/src/sdk/interface/ConnectionImpl.h @@ -106,6 +106,12 @@ class ConnectionImpl : public Connection { Status DropPartition(const PartitionParam& param) override; + Status + GetConfig(const std::string& node_name, std::string& value) const override; + + Status + SetConfig(const std::string& node_name, const std::string& value) const override; + private: std::shared_ptr client_proxy_; }; diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index c770ff7be6..1e82b6a863 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -33,7 +33,7 @@ namespace milvus { namespace server { -constexpr uint64_t GB = 1UL << 30; +constexpr int64_t GB = 1UL << 30; static const std::unordered_map milvus_config_version_map({{"0.6.0", "0.1"}}); @@ -400,6 +400,92 @@ Config::PrintAll() { PrintConfigSection(CONFIG_GPU_RESOURCE); } +Status +Config::GetConfigCli(const std::string& parent_key, const std::string& child_key, std::string& value) { + if (!ConfigNodeValid(parent_key, child_key)) { + std::string str = "Config node invalid: " + parent_key + CONFIG_NODE_DELIMITER + child_key; + return Status(SERVER_UNEXPECTED_ERROR, str); + } + return GetConfigValueInMem(parent_key, child_key, value); +} + +Status +Config::SetConfigCli(const std::string& parent_key, const std::string& child_key, const std::string& value) { + if (!ConfigNodeValid(parent_key, child_key)) { + std::string str = "Config node invalid: " + parent_key + CONFIG_NODE_DELIMITER + child_key; + return Status(SERVER_UNEXPECTED_ERROR, str); + } + if (parent_key == CONFIG_SERVER) { + return Status(SERVER_UNSUPPORTED_ERROR, "Not support set server_config"); + } else if (parent_key == CONFIG_DB) { + return Status(SERVER_UNSUPPORTED_ERROR, "Not support set db_config"); + } else if (parent_key == CONFIG_METRIC) { + return Status(SERVER_UNSUPPORTED_ERROR, "Not support set metric_config"); + } else if (parent_key == CONFIG_CACHE) { + if (child_key == CONFIG_CACHE_CPU_CACHE_CAPACITY) { + return SetCacheConfigCpuCacheCapacity(value); + } else if (child_key == CONFIG_CACHE_CPU_CACHE_THRESHOLD) { + return SetCacheConfigCpuCacheThreshold(value); + } else if (child_key == CONFIG_CACHE_CACHE_INSERT_DATA) { + return SetCacheConfigCacheInsertData(value); + } + } else if (parent_key == CONFIG_ENGINE) { + if (child_key == CONFIG_ENGINE_USE_BLAS_THRESHOLD) { + return SetEngineConfigUseBlasThreshold(value); + } else if (child_key == CONFIG_ENGINE_OMP_THREAD_NUM) { + return SetEngineConfigOmpThreadNum(value); +#ifdef MILVUS_GPU_VERSION + } else if (child_key == CONFIG_ENGINE_GPU_SEARCH_THRESHOLD) { + return SetEngineConfigGpuSearchThreshold(value); +#endif + } +#ifdef MILVUS_GPU_VERSION + } else if (parent_key == CONFIG_GPU_RESOURCE) { + if (child_key == CONFIG_GPU_RESOURCE_ENABLE) { + return SetGpuResourceConfigEnable(value); + } else if (child_key == CONFIG_GPU_RESOURCE_CACHE_CAPACITY) { + return SetGpuResourceConfigCacheCapacity(value); + } else if (child_key == CONFIG_GPU_RESOURCE_CACHE_THRESHOLD) { + return SetGpuResourceConfigCacheThreshold(value); + } else if (child_key == CONFIG_GPU_RESOURCE_SEARCH_RESOURCES) { + return SetGpuResourceConfigSearchResources(value); + } else if (child_key == CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES) { + return SetGpuResourceConfigBuildIndexResources(value); + } +#endif + } else if (parent_key == CONFIG_TRACING) { + return Status(SERVER_UNSUPPORTED_ERROR, "Not support set tracing_config"); + } +} + +Status +Config::HandleConfigCli(std::string& result, const std::string& cmd) { + std::vector tokens; + std::vector nodes; + server::StringHelpFunctions::SplitStringByDelimeter(cmd, " ", tokens); + if (tokens[0] == "get") { + if (tokens.size() != 2) { + return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + } + server::StringHelpFunctions::SplitStringByDelimeter(tokens[1], CONFIG_NODE_DELIMITER, nodes); + if (nodes.size() != 2) { + return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + } + return GetConfigCli(nodes[0], nodes[1], result); + } else if (tokens[0] == "set") { + if (tokens.size() != 3) { + return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + } + server::StringHelpFunctions::SplitStringByDelimeter(tokens[1], CONFIG_NODE_DELIMITER, nodes); + if (nodes.size() != 2) { + return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + } + return SetConfigCli(nodes[0], nodes[1], tokens[2]); + } else { + return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + } +} + //////////////////////////////////////////////////////////////////////////////// Status Config::CheckConfigVersion(const std::string& value) { @@ -804,6 +890,17 @@ Config::GetConfigNode(const std::string& name) { return GetConfigRoot().GetChild(name); } +bool +Config::ConfigNodeValid(const std::string& parent_key, const std::string& child_key) { + if (config_map_.find(parent_key) == config_map_.end()) { + return false; + } + if (config_map_[parent_key].count(child_key) == 0) { + return false; + } + return true; +} + Status Config::GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value) { std::lock_guard lock(mutex_); @@ -815,10 +912,11 @@ Config::GetConfigValueInMem(const std::string& parent_key, const std::string& ch return Status(SERVER_UNEXPECTED_ERROR, "key not exist"); } -void +Status Config::SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value) { std::lock_guard lock(mutex_); config_map_[parent_key][child_key] = value; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////// diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 9939779e0a..9062070e79 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -28,6 +28,7 @@ namespace milvus { namespace server { +static const char* CONFIG_NODE_DELIMITER = "."; static const char* CONFIG_VERSION = "version"; /* server config */ @@ -56,6 +57,7 @@ static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4"; static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table"; +static const char* CONFIG_DB_PRELOAD_TABLE_DEFAULT = ""; /* cache config */ static const char* CONFIG_CACHE = "cache_config"; @@ -120,18 +122,26 @@ class Config { ResetDefaultConfig(); void PrintAll(); + Status + HandleConfigCli(std::string& result, const std::string& cmd); private: ConfigNode& GetConfigRoot(); ConfigNode& GetConfigNode(const std::string& name); + bool + ConfigNodeValid(const std::string& parent_key, const std::string& child_key); Status GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value); - void + Status SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value); void PrintConfigSection(const std::string& config_node_name); + Status + GetConfigCli(const std::string& parent_key, const std::string& child_key, std::string& value); + Status + SetConfigCli(const std::string& parent_key, const std::string& child_key, const std::string& value); /////////////////////////////////////////////////////////////////////////// Status diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 791876ee8b..a85cd7d910 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -35,6 +35,8 @@ static constexpr uint64_t GB = MB * 1024; } // namespace +namespace ms = milvus::server; + TEST_F(ConfigTest, CONFIG_TEST) { milvus::server::ConfigMgr* config_mgr = milvus::server::YamlConfigMgr::GetInstance(); @@ -307,6 +309,154 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { #endif } +std::string gen_get_command(const std::string& parent_node, const std::string& child_node) { + std::string cmd = "get " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node; + return cmd; +} + +std::string gen_set_command(const std::string& parent_node, const std::string& child_node, const std::string& value) { + std::string cmd = "set " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node + " " + value; + return cmd; +} + +TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { + std::string config_path(CONFIG_PATH); + milvus::server::Config& config = milvus::server::Config::GetInstance(); + milvus::Status s; + + std::string get_cmd, set_cmd; + std::string result, dummy; + + /* server config */ + std::string server_addr = "192.168.1.155"; + get_cmd = gen_get_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS); + set_cmd = gen_set_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS, server_addr); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_FALSE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + + /* db config */ + std::string db_primary_path = "/home/zilliz"; + get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_PRIMARY_PATH); + set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_PRIMARY_PATH, db_primary_path); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_FALSE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + + /* metric config */ + std::string metric_enable_monitor = "false"; + get_cmd = gen_get_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR); + set_cmd = gen_set_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR, metric_enable_monitor); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_FALSE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + + /* cache config */ + std::string cache_cpu_cache_capacity = "5"; + get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY); + set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY, cache_cpu_cache_capacity); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == cache_cpu_cache_capacity); + + std::string cache_cpu_cache_threshold = "0.1"; + get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD); + set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD, cache_cpu_cache_threshold); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(result == cache_cpu_cache_threshold); + + std::string cache_insert_data = "true"; + get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA); + set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA, cache_insert_data); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(result == cache_insert_data); + + /* engine config */ + std::string engine_use_blas_threshold = "50"; + get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD); + set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD, engine_use_blas_threshold); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == engine_use_blas_threshold); + + std::string engine_omp_thread_num = "8"; + get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM); + set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM, engine_omp_thread_num); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == engine_omp_thread_num); + +#ifdef MILVUS_GPU_VERSION + std::string engine_gpu_search_threshold = "800"; + get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD); + set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, engine_gpu_search_threshold); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == engine_gpu_search_threshold); + + /* gpu resource config */ + std::string resource_enable_gpu = "true"; + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE); + set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE, resource_enable_gpu); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == resource_enable_gpu); + + std::string gpu_cache_capacity = "1"; + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY); + set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY, gpu_cache_capacity); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == gpu_cache_capacity); + + std::string gpu_cache_threshold = "0.2"; + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD); + set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, gpu_cache_threshold); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(result == gpu_cache_threshold); + + std::string search_resources = "gpu0"; + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES); + set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, search_resources); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == search_resources); + + std::string build_index_resources = "gpu0"; + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES); + set_cmd = + gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, build_index_resources); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == build_index_resources); +#endif +} + TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { std::string config_path(CONFIG_PATH); milvus::server::Config& config = milvus::server::Config::GetInstance(); From 21e986d130c12ae5bd82b8ca829622cef6f6489d Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Sat, 14 Dec 2019 17:01:56 +0800 Subject: [PATCH 2/3] support config cli (#742) * #665 support get/set config via CLI * #665 support get/set config via CLI * #665 add unittest for config CLI * #665 remove config_node_map_ * #665 remove config_node_map_ * #665 fix clang-format * #665 fix clang-format * #665 update changelog * #665 code clean * #665 update API interface * #665 update API interface * #665 handle server status * #665 handle unknown command * #665 update debug log * #665 support get/set config via CLI * #665 support get/set config via CLI * #665 add unittest for config CLI * #665 remove config_node_map_ * #665 remove config_node_map_ * #665 fix clang-format * #665 fix clang-format * #665 update changelog * #665 code clean * #665 update API interface * #665 update API interface * #665 handle server status * #665 handle unknown command * #665 update debug log --- core/src/server/delivery/request/CmdRequest.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/src/server/delivery/request/CmdRequest.cpp b/core/src/server/delivery/request/CmdRequest.cpp index 3bf779f1b3..b2434076eb 100644 --- a/core/src/server/delivery/request/CmdRequest.cpp +++ b/core/src/server/delivery/request/CmdRequest.cpp @@ -38,9 +38,12 @@ Status CmdRequest::OnExecute() { std::string hdr = "CmdRequest(cmd=" + cmd_ + ")"; TimeRecorderAuto rc(hdr); + Status stat = Status::OK(); if (cmd_ == "version") { result_ = MILVUS_VERSION; + } else if (cmd_ == "status") { + result_ = "OK"; } else if (cmd_ == "tasktable") { result_ = scheduler::ResMgrInst::GetInstance()->DumpTaskTables(); } else if (cmd_ == "mode") { @@ -49,11 +52,14 @@ CmdRequest::OnExecute() { #else result_ = "CPU"; #endif + } else if (cmd_.substr(0, 3) == "set" || cmd_.substr(0, 3) == "get") { + server::Config& config = server::Config::GetInstance(); + stat = config.HandleConfigCli(result_, cmd_); } else { - result_ = "OK"; + result_ = "Unknown command"; } - return Status::OK(); + return stat; } } // namespace server From 952f34352a909ef02238fee75ca84921615ff509 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 16 Dec 2019 14:07:20 +0800 Subject: [PATCH 3/3] enable build milvus on centos7 (#769) * enable build milvus on centos7 * Update build enviroment Centos7 dockerfile * Update Dockerfile * Update Dockerfile * Update Dockerfile * Update Dockerfile * Update Dockerfile * Update Dockerfile * Update Dockerfile * add centos7_build_deps.sh * add centos7 build cpu version enviroment * add centos7 on github actions * fix bug * fix bug * fix bug * update ci/docker/centos-7-core.dockerfile * fix github actions bug * update centos7 case on github actions * update docker-compose.yml * debug centos case on github actions * debug centos case on github actions * add centos7 deploy dockerfile --- .env | 1 + .github/workflows/core.yml | 30 +++++++++++++++++ ci/docker/centos-7-core.dockerfile | 18 ++++++++++ core/centos7_build_deps.sh | 11 +++++++ .../index/cmake/ThirdPartyPackagesCore.cmake | 7 ++-- core/unittest/server/test_config.cpp | 5 +-- docker-compose.yml | 33 +++++++++++++++---- docker/build_env/cpu/centos7/Dockerfile | 20 +++++++++++ .../cpu/centos7/docker-entrypoint.sh | 10 ++++++ docker/build_env/gpu/centos7/Dockerfile | 22 +++++++++++++ .../gpu/centos7/docker-entrypoint.sh | 10 ++++++ docker/deploy/cpu/centos7/Dockerfile | 17 ++++++++++ .../deploy/cpu/centos7/docker-entrypoint.sh | 9 +++++ docker/deploy/gpu/centos7/Dockerfile | 19 +++++++++++ .../deploy/gpu/centos7/docker-entrypoint.sh | 9 +++++ 15 files changed, 210 insertions(+), 11 deletions(-) create mode 100644 ci/docker/centos-7-core.dockerfile create mode 100755 core/centos7_build_deps.sh create mode 100644 docker/build_env/cpu/centos7/Dockerfile create mode 100755 docker/build_env/cpu/centos7/docker-entrypoint.sh create mode 100644 docker/build_env/gpu/centos7/Dockerfile create mode 100755 docker/build_env/gpu/centos7/docker-entrypoint.sh create mode 100644 docker/deploy/cpu/centos7/Dockerfile create mode 100755 docker/deploy/cpu/centos7/docker-entrypoint.sh create mode 100644 docker/deploy/gpu/centos7/Dockerfile create mode 100755 docker/deploy/gpu/centos7/docker-entrypoint.sh diff --git a/.env b/.env index bcb772c2de..3299802e25 100644 --- a/.env +++ b/.env @@ -2,4 +2,5 @@ REPO=milvusdb/milvus-dev MILVUS_INSTALL_PREFIX=/var/lib/milvus ARCH=amd64 UBUNTU=18.04 +CENTOS=7 CUDA=10.1 \ No newline at end of file diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index ab26f0b189..c1c09e01df 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -40,3 +40,33 @@ jobs: docker login -u ${{ secrets.DOCKERHUB_USER }} \ -p ${{ secrets.DOCKERHUB_TOKEN }} docker-compose push ubuntu-core + + centos: + name: AMD64 CentOS ${{ matrix.centos }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + centos: [7] + env: + CENTOS: ${{ matrix.centos }} + steps: + - name: Checkout Milvus + uses: actions/checkout@v1 + - name: Docker Pull + shell: bash + run: | + docker-compose pull --ignore-pull-failures db + docker-compose pull --ignore-pull-failures centos-core + - name: Docker Build + run: | + docker-compose run --use-aliases -d db + docker-compose run centos-core + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' + continue-on-error: true + shell: bash + run: | + docker login -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push centos-core diff --git a/ci/docker/centos-7-core.dockerfile b/ci/docker/centos-7-core.dockerfile new file mode 100644 index 0000000000..5bb7ca385f --- /dev/null +++ b/ci/docker/centos-7-core.dockerfile @@ -0,0 +1,18 @@ +ARG arch=amd64 +FROM ${arch}/centos:7 + +# pipefail is enabled for proper error detection in the `wget` +# step +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl which && \ + wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ + yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ + devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra mysql lcov \ + && \ + rm -rf /var/cache/yum/* + +RUN echo "source scl_source enable devtoolset-7" >> /etc/profile.d/devtoolset-7.sh +RUN echo "source scl_source enable llvm-toolset-7.0" >> /etc/profile.d/llvm-toolset-7.sh + +ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin" diff --git a/core/centos7_build_deps.sh b/core/centos7_build_deps.sh new file mode 100755 index 0000000000..f415e359af --- /dev/null +++ b/core/centos7_build_deps.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +sudo yum install -y epel-release centos-release-scl-rh && sudo yum install -y wget curl which && \ +sudo wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | sudo tar --strip-components=1 -xz -C /usr/local && \ +sudo yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ +devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra lcov + +echo "source scl_source enable devtoolset-7" | sudo tee -a /etc/profile.d/devtoolset-7.sh +echo "source scl_source enable llvm-toolset-7.0" | sudo tee -a /etc/profile.d/llvm-toolset-7.sh +echo "export CLANG_TOOLS_PATH=/opt/rh/llvm-toolset-7.0/root/usr/bin" | sudo tee -a /etc/profile.d/llvm-toolset-7.sh + diff --git a/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/core/src/index/cmake/ThirdPartyPackagesCore.cmake index 9c9187d2cc..9ae0391f54 100644 --- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -270,10 +270,10 @@ set(ARROW_PREFIX "${INDEX_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") macro(build_arrow) message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source") set(ARROW_STATIC_LIB_NAME arrow) - set(ARROW_STATIC_LIB - "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) set(ARROW_LIB_DIR "${ARROW_PREFIX}/lib") + set(ARROW_STATIC_LIB + "${ARROW_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") set(ARROW_CMAKE_ARGS @@ -282,6 +282,7 @@ macro(build_arrow) -DARROW_BUILD_SHARED=OFF -DARROW_USE_GLOG=OFF -DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX} + -DCMAKE_INSTALL_LIBDIR=${ARROW_LIB_DIR} -DARROW_CUDA=OFF -DARROW_FLIGHT=OFF -DARROW_GANDIVA=OFF diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index a85cd7d910..3c27572f1a 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -17,6 +17,7 @@ #include #include +#include #include "config/YamlConfigMgr.h" #include "server/Config.h" @@ -69,9 +70,9 @@ TEST_F(ConfigTest, CONFIG_TEST) { server_config.SetValue("float_test", "2.5"); double dbl = server_config.GetDoubleValue("float_test"); - ASSERT_LE(abs(dbl - 2.5), std::numeric_limits::epsilon()); + ASSERT_LE(std::fabs(dbl - 2.5), std::numeric_limits::epsilon()); float flt = server_config.GetFloatValue("float_test"); - ASSERT_LE(abs(flt - 2.5), std::numeric_limits::epsilon()); + ASSERT_LE(std::fabs(flt - 2.5), std::numeric_limits::epsilon()); server_config.SetValue("bool_test", "true"); bool blt = server_config.GetBoolValue("bool_test"); diff --git a/docker-compose.yml b/docker-compose.yml index 19483016b8..4eda37f488 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,7 @@ version: '3.5' volumes: amd64-ubuntu-18.04-cache: + amd64-centos-7-cache: x-ccache: &ccache CCACHE_COMPILERCHECK: content @@ -10,11 +11,6 @@ x-ccache: &ccache CCACHE_MAXSIZE: 2G CCACHE_DIR: /build/ccache -x-command: &cpp-command > - /bin/bash -c " - /milvus/ci/scripts/build.sh -o ${MILVUS_INSTALL_PREFIX} -l -u -c - /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" - services: db: image: mysql:5.6 @@ -42,7 +38,32 @@ services: - ${ARCH}-ubuntu-${UBUNTU}-cache:/build:delegated networks: - milvus - command: *cpp-command + command: &ubuntu-command > + /bin/bash -c " + /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c + /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" + + centos-core: + image: ${REPO}:${ARCH}-centos-${CENTOS}-core + build: + context: . + dockerfile: ci/docker/centos-${CENTOS}-core.dockerfile + cache_from: + - ${REPO}:${ARCH}-centos-${CENTOS}-core + shm_size: 2G + environment: + <<: *ccache + volumes: ¢os-volumes + - .:/milvus:delegated + - ${ARCH}-centos-${CENTOS}-cache:/build:delegated + networks: + - milvus + command: ¢os-command > + /bin/bash -c " + source scl_source enable devtoolset-7 + source scl_source enable llvm-toolset-7.0 + /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c + /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" networks: milvus: diff --git a/docker/build_env/cpu/centos7/Dockerfile b/docker/build_env/cpu/centos7/Dockerfile new file mode 100644 index 0000000000..6b3817f849 --- /dev/null +++ b/docker/build_env/cpu/centos7/Dockerfile @@ -0,0 +1,20 @@ +FROM centos:centos7 + +RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl which && \ + wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ + yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ + devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra mysql lcov \ + && \ + rm -rf /var/cache/yum/* + +RUN echo "source scl_source enable devtoolset-7" >> /etc/profile.d/devtoolset-7.sh +RUN echo "source scl_source enable llvm-toolset-7.0" >> /etc/profile.d/llvm-toolset-7.sh + +ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin" + +COPY docker-entrypoint.sh /app/docker-entrypoint.sh + +WORKDIR /root + +ENTRYPOINT [ "/app/docker-entrypoint.sh" ] +CMD [ "start" ] \ No newline at end of file diff --git a/docker/build_env/cpu/centos7/docker-entrypoint.sh b/docker/build_env/cpu/centos7/docker-entrypoint.sh new file mode 100755 index 0000000000..1e85e7e9e1 --- /dev/null +++ b/docker/build_env/cpu/centos7/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +if [ "$1" = 'start' ]; then + tail -f /dev/null +fi + +exec "$@" + diff --git a/docker/build_env/gpu/centos7/Dockerfile b/docker/build_env/gpu/centos7/Dockerfile new file mode 100644 index 0000000000..fc800d1b5d --- /dev/null +++ b/docker/build_env/gpu/centos7/Dockerfile @@ -0,0 +1,22 @@ +FROM nvidia/cuda:10.1-devel-centos7 + +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility + +RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl which && \ + wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ + yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ + devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra mysql lcov \ + && \ + rm -rf /var/cache/yum/* + +RUN echo "source scl_source enable devtoolset-7" >> /etc/profile.d/devtoolset-7.sh +RUN echo "source scl_source enable llvm-toolset-7.0" >> /etc/profile.d/llvm-toolset-7.sh + +ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin" + +COPY docker-entrypoint.sh /app/docker-entrypoint.sh + +WORKDIR /root + +ENTRYPOINT [ "/app/docker-entrypoint.sh" ] +CMD [ "start" ] diff --git a/docker/build_env/gpu/centos7/docker-entrypoint.sh b/docker/build_env/gpu/centos7/docker-entrypoint.sh new file mode 100755 index 0000000000..1e85e7e9e1 --- /dev/null +++ b/docker/build_env/gpu/centos7/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +if [ "$1" = 'start' ]; then + tail -f /dev/null +fi + +exec "$@" + diff --git a/docker/deploy/cpu/centos7/Dockerfile b/docker/deploy/cpu/centos7/Dockerfile new file mode 100644 index 0000000000..f1e4169d41 --- /dev/null +++ b/docker/deploy/cpu/centos7/Dockerfile @@ -0,0 +1,17 @@ +FROM centos:centos7 + +RUN yum install -y epel-release && \ + yum install -y libgomp libgfortran4 mysql-devel && \ + rm -rf /var/cache/yum/* + +COPY ./milvus /var/lib/milvus +COPY ./docker-entrypoint.sh /var/lib/milvus +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/var/lib/milvus/lib" + +WORKDIR /var/lib/milvus + +ENTRYPOINT [ "/var/lib/milvus/docker-entrypoint.sh" ] + +CMD [ "start" ] + +EXPOSE 19530 diff --git a/docker/deploy/cpu/centos7/docker-entrypoint.sh b/docker/deploy/cpu/centos7/docker-entrypoint.sh new file mode 100755 index 0000000000..830b394eb7 --- /dev/null +++ b/docker/deploy/cpu/centos7/docker-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +if [ "$1" == 'start' ]; then + cd /var/lib/milvus/scripts && ./start_server.sh +fi + +exec "$@" diff --git a/docker/deploy/gpu/centos7/Dockerfile b/docker/deploy/gpu/centos7/Dockerfile new file mode 100644 index 0000000000..c43debfe71 --- /dev/null +++ b/docker/deploy/gpu/centos7/Dockerfile @@ -0,0 +1,19 @@ +FROM nvidia/cuda:10.1-devel-centos7 + +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility + +RUN yum install -y epel-release && \ + yum install -y libgomp libgfortran4 mysql-devel && \ + rm -rf /var/cache/yum/* + +COPY ./milvus /var/lib/milvus +COPY ./docker-entrypoint.sh /var/lib/milvus +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/var/lib/milvus/lib" + +WORKDIR /var/lib/milvus + +ENTRYPOINT [ "/var/lib/milvus/docker-entrypoint.sh" ] + +CMD [ "start" ] + +EXPOSE 19530 diff --git a/docker/deploy/gpu/centos7/docker-entrypoint.sh b/docker/deploy/gpu/centos7/docker-entrypoint.sh new file mode 100755 index 0000000000..830b394eb7 --- /dev/null +++ b/docker/deploy/gpu/centos7/docker-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +if [ "$1" == 'start' ]; then + cd /var/lib/milvus/scripts && ./start_server.sh +fi + +exec "$@"