diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c44e9cbe8..3b211e3fa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version - \#458 - Index data is not compatible between 0.5 and 0.6 - \#465 - Server hang caused by searching with nsg index +- \#485 - Increase code coverage rate - \#486 - gpu no usage during index building - \#497 - CPU-version search performance decreased - \#504 - The code coverage rate of core/src/scheduler/optimizer is too low @@ -37,6 +38,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#561 - Milvus server should report exception/error message or terminate on mysql metadata backend error - \#596 - Frequently insert operation cost too much disk space - \#599 - Build index log is incorrect +- \#602 - Optimizer specify wrong gpu_id ## Feature - \#12 - Pure CPU version for Milvus @@ -48,6 +50,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#420 - Update shards merge part to match v0.5.3 - \#488 - Add log in scheduler/optimizer - \#502 - C++ SDK support IVFPQ and SPTAG +- \#560 - Add version in server config file ## Improvement - \#255 - Add ivfsq8 test report detailed version diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 41889f5cef..40b8107d39 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -1,5 +1,7 @@ # Default values are used when you make no changes to the following parameters. +version: 0.1 # config version + server_config: address: 0.0.0.0 # milvus server ip address (IPv4) port: 19530 # milvus server port, must in range [1025, 65534] diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 531c633da7..678915ca9d 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -1,5 +1,7 @@ # Default values are used when you make no changes to the following parameters. +version: 0.1 # config version + server_config: address: 0.0.0.0 # milvus server ip address (IPv4) port: 19530 # milvus server port, must in range [1025, 65534] diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 397ca89d68..5a3d6e5e2a 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -112,10 +112,12 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { index = GetVecIndexFactory(IndexType::NSG_MIX); break; } +#ifdef CUSTOMIZATION case EngineType::FAISS_IVFSQ8H: { index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_HYBRID); break; } +#endif case EngineType::FAISS_PQ: { #ifdef MILVUS_CPU_VERSION index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_CPU); diff --git a/core/src/scheduler/optimizer/FaissFlatPass.cpp b/core/src/scheduler/optimizer/FaissFlatPass.cpp index 15ddc62f55..f7377d22ef 100644 --- a/core/src/scheduler/optimizer/FaissFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissFlatPass.cpp @@ -60,7 +60,7 @@ FaissFlatPass::Run(const TaskPtr& task) { auto best_device_id = count_ % gpus.size(); SERVER_LOG_DEBUG << "FaissFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id << " to search!"; count_++; - res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpus[best_device_id]); } auto label = std::make_shared(res_ptr); task->label() = label; diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp index 70909e8cd1..b8fe778432 100644 --- a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp @@ -63,7 +63,7 @@ FaissIVFFlatPass::Run(const TaskPtr& task) { SERVER_LOG_DEBUG << "FaissIVFFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id << " to search!"; count_++; - res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpus[best_device_id]); } auto label = std::make_shared(res_ptr); task->label() = label; diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index 7bcee6b07e..ad4184187d 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -61,7 +61,7 @@ FaissIVFSQ8HPass::Run(const TaskPtr& task) { SERVER_LOG_DEBUG << "FaissIVFSQ8HPass: nq > gpu_search_threshold, specify gpu" << best_device_id << " to search!"; count_++; - res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpus[best_device_id]); } auto label = std::make_shared(res_ptr); task->label() = label; diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp index 7df917000c..280b024894 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp @@ -63,7 +63,7 @@ FaissIVFSQ8Pass::Run(const TaskPtr& task) { SERVER_LOG_DEBUG << "FaissIVFSQ8Pass: nq > gpu_search_threshold, specify gpu" << best_device_id << " to search!"; count_++; - res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpus[best_device_id]); } auto label = std::make_shared(res_ptr); task->label() = label; diff --git a/core/src/sdk/include/MilvusApi.h b/core/src/sdk/include/MilvusApi.h index 5c7736d4e2..0937ce2f8e 100644 --- a/core/src/sdk/include/MilvusApi.h +++ b/core/src/sdk/include/MilvusApi.h @@ -85,18 +85,18 @@ struct RowRecord { * @brief TopK query result */ struct QueryResult { - std::vector ids; - std::vector distances; + std::vector ids; ///< Query ids result + std::vector distances; ///< Query distances result }; -using TopKQueryResult = std::vector; +using TopKQueryResult = std::vector; ///< Topk query result /** * @brief index parameters */ struct IndexParam { - std::string table_name; - IndexType index_type; - int32_t nlist; + std::string table_name; ///< Table name for create index + IndexType index_type; ///< Create index type + int32_t nlist; ///< Index nlist }; /** @@ -142,8 +142,8 @@ class Connection { /** * @brief Connect * - * Connect function should be called before any operations - * Server will be connected after Connect return OK + * This method is used to connect server. + * Connect function should be called before any operations. * * @param param, use to provide server information * @@ -156,10 +156,10 @@ class Connection { /** * @brief Connect * - * Connect function should be called before any operations - * Server will be connected after Connect return OK + * This method is used to connect server. + * Connect function should be called before any operations. * - * @param uri, use to provide server information, example: milvus://ipaddress:port + * @param uri, use to provide server uri, example: milvus://ipaddress:port * * @return Indicate if connect is successful */ @@ -169,7 +169,7 @@ class Connection { /** * @brief connected * - * Connection status. + * This method is used to test whether server is connected. * * @return Indicate if connection status */ @@ -179,7 +179,7 @@ class Connection { /** * @brief Disconnect * - * Server will be disconnected after Disconnect return OK + * This method is used to disconnect server. * * @return Indicate if disconnect is successful */ @@ -189,7 +189,7 @@ class Connection { /** * @brief Create table method * - * This method is used to create table + * This method is used to create table. * * @param param, use to provide table information to be created. * @@ -201,7 +201,7 @@ class Connection { /** * @brief Test table existence method * - * This method is used to create table + * This method is used to create table. * * @param table_name, target table's name. * @@ -211,13 +211,13 @@ class Connection { HasTable(const std::string& table_name) = 0; /** - * @brief Delete table method + * @brief Drop table method * - * This method is used to delete table(and its partitions). + * This method is used to drop table(and its partitions). * * @param table_name, target table's name. * - * @return Indicate if table is delete successfully. + * @return Indicate if table is drop successfully. */ virtual Status DropTable(const std::string& table_name) = 0; @@ -239,14 +239,17 @@ class Connection { CreateIndex(const IndexParam& index_param) = 0; /** - * @brief Add vector to table + * @brief Insert vector to table * - * This method is used to add vector array to table. + * This method is used to insert vector array to table. * * @param table_name, target table's name. * @param partition_tag, target partition's tag, keep empty if no partition. * @param record_array, vector array is inserted. - * @param id_array, after inserted every vector is given a id. + * @param id_array, + * specify id for each vector, + * if this array is empty, milvus will generate unique id for each vector, + * and return all ids by this parameter. * * @return Indicate if vector array are inserted successfully */ @@ -259,11 +262,12 @@ class Connection { * * This method is used to query vector in table. * - * @param table_name, target table's name, keep empty if no partition. - * @param partition_tags, target partitions. + * @param table_name, target table's name. + * @param partition_tags, target partitions, keep empty if no partition. * @param query_record_array, all vector are going to be queried. - * @param query_range_array, time ranges, if not specified, will search in whole table + * @param query_range_array, [deprecated] time ranges, if not specified, will search in whole table * @param topk, how many similarity vectors will be searched. + * @param nprobe, the number of centroids choose to search. * @param topk_query_result_array, result array. * * @return Indicate if query is successful. @@ -304,7 +308,7 @@ class Connection { * * This method is used to list all tables. * - * @param table_array, all tables are push into the array. + * @param table_array, all tables in database. * * @return Indicate if this operation is successful. */ @@ -346,12 +350,13 @@ class Connection { * * This method is internal used. * - * @return Server status. + * @return Task information in tasktables. */ virtual std::string DumpTaskTables() const = 0; /** + * [deprecated] * @brief delete tables by date range * * This method is used to delete table data by date range. diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 95bab84391..cd9b264fcd 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "config/YamlConfigMgr.h" @@ -33,6 +34,8 @@ namespace server { constexpr uint64_t GB = 1UL << 30; +static const std::unordered_map milvus_config_version_map({{"0.6.0", "0.1"}}); + Config& Config::GetInstance() { static Config config_inst; @@ -69,6 +72,12 @@ Status Config::ValidateConfig() { Status s; + std::string config_version; + s = GetConfigVersion(config_version); + if (!s.ok()) { + return s; + } + /* server config */ std::string server_addr; s = GetServerConfigAddress(server_addr); @@ -383,6 +392,16 @@ Config::PrintAll() { } //////////////////////////////////////////////////////////////////////////////// +Status +Config::CheckConfigVersion(const std::string& value) { + if (milvus_config_version_map.at(MILVUS_VERSION) != value) { + std::string msg = "Invalid config version: " + value + + ". Expected config version: " + milvus_config_version_map.at(MILVUS_VERSION); + return Status(SERVER_INVALID_ARGUMENT, msg); + } + return Status::OK(); +} + Status Config::CheckServerConfigAddress(const std::string& value) { if (!ValidationUtil::ValidateIpAddress(value).ok()) { @@ -766,10 +785,14 @@ Config::CheckGpuResourceConfigBuildIndexResources(const std::vector //////////////////////////////////////////////////////////////////////////////// ConfigNode& -Config::GetConfigNode(const std::string& name) { +Config::GetConfigRoot() { ConfigMgr* mgr = YamlConfigMgr::GetInstance(); - ConfigNode& root_node = mgr->GetRootNode(); - return root_node.GetChild(name); + return mgr->GetRootNode(); +} + +ConfigNode& +Config::GetConfigNode(const std::string& name) { + return GetConfigRoot().GetChild(name); } Status @@ -816,6 +839,12 @@ Config::GetConfigSequenceStr(const std::string& parent_key, const std::string& c return value; } +Status +Config::GetConfigVersion(std::string& value) { + value = GetConfigRoot().GetValue(CONFIG_VERSION); + return CheckConfigVersion(value); +} + Status Config::GetServerConfigAddress(std::string& value) { value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 281a832d57..4361cd1e37 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -28,6 +28,8 @@ namespace milvus { namespace server { +static const char* CONFIG_VERSION = "version"; + /* server config */ static const char* CONFIG_SERVER = "server_config"; static const char* CONFIG_SERVER_ADDRESS = "address"; @@ -115,6 +117,8 @@ class Config { PrintAll(); private: + ConfigNode& + GetConfigRoot(); ConfigNode& GetConfigNode(const std::string& name); Status @@ -125,6 +129,9 @@ class Config { PrintConfigSection(const std::string& config_node_name); /////////////////////////////////////////////////////////////////////////// + Status + CheckConfigVersion(const std::string& value); + /* server config */ Status CheckServerConfigAddress(const std::string& value); @@ -193,6 +200,8 @@ class Config { std::string GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim = ",", const std::string& default_value = ""); + Status + GetConfigVersion(std::string& value); public: /* server config */ diff --git a/core/unittest/db/test_engine.cpp b/core/unittest/db/test_engine.cpp index 67070df41b..e0edd05537 100644 --- a/core/unittest/db/test_engine.cpp +++ b/core/unittest/db/test_engine.cpp @@ -59,6 +59,29 @@ TEST_F(EngineTest, FACTORY_TEST) { ASSERT_TRUE(engine_ptr != nullptr); } + + { + auto engine_ptr = milvus::engine::EngineFactory::Build( + 512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_PQ, milvus::engine::MetricType::IP, 1024); + + ASSERT_TRUE(engine_ptr != nullptr); + } + + { + auto engine_ptr = milvus::engine::EngineFactory::Build( + 512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT, + milvus::engine::MetricType::L2, 1024); + + ASSERT_TRUE(engine_ptr != nullptr); + } + + { + auto engine_ptr = milvus::engine::EngineFactory::Build( + 512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT, + milvus::engine::MetricType::L2, 1024); + + ASSERT_TRUE(engine_ptr != nullptr); + } } TEST_F(EngineTest, ENGINE_IMPL_TEST) { @@ -69,7 +92,7 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) { std::vector data; std::vector ids; - const int row_count = 10000; + const int row_count = 500; data.reserve(row_count * dimension); ids.reserve(row_count); for (int64_t i = 0; i < row_count; i++) { @@ -95,5 +118,8 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) { // ASSERT_TRUE(status.ok()); auto engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IVFSQ8); + engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_3", milvus::engine::EngineType::FAISS_PQ); + engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_4", milvus::engine::EngineType::SPTAG_KDT); + engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_5", milvus::engine::EngineType::SPTAG_BKT); // ASSERT_TRUE(status.ok()); } diff --git a/core/unittest/server/utils.cpp b/core/unittest/server/utils.cpp index 0545d34575..6fb424356d 100644 --- a/core/unittest/server/utils.cpp +++ b/core/unittest/server/utils.cpp @@ -28,6 +28,8 @@ namespace { static const char* VALID_CONFIG_STR = "# Default values are used when you make no changes to the following parameters.\n" "\n" + "version: 0.1" + "\n" "server_config:\n" " address: 0.0.0.0 # milvus server ip address (IPv4)\n" " port: 19530 # port range: 1025 ~ 65534\n"