From 4cf4cc7789f66e553f7e2752bb1ecc80fa45045d Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Thu, 28 Nov 2019 18:48:34 +0800 Subject: [PATCH 1/3] Update some annotation and description in MilvusApi.h --- core/src/sdk/include/MilvusApi.h | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/core/src/sdk/include/MilvusApi.h b/core/src/sdk/include/MilvusApi.h index 5c7736d4e2..22155db3be 100644 --- a/core/src/sdk/include/MilvusApi.h +++ b/core/src/sdk/include/MilvusApi.h @@ -85,18 +85,18 @@ struct RowRecord { * @brief TopK query result */ struct QueryResult { - std::vector ids; - std::vector distances; + std::vector ids; ///< Query ids result + std::vector distances; ///< Query distances result }; -using TopKQueryResult = std::vector; +using TopKQueryResult = std::vector; ///< Topk query result /** * @brief index parameters */ struct IndexParam { - std::string table_name; - IndexType index_type; - int32_t nlist; + std::string table_name; ///< Table name for create index + IndexType index_type; ///< Create index type + int32_t nlist; ///< Index nlist }; /** @@ -211,13 +211,13 @@ class Connection { HasTable(const std::string& table_name) = 0; /** - * @brief Delete table method + * @brief Drop table method * - * This method is used to delete table(and its partitions). + * This method is used to drop table(and its partitions). * * @param table_name, target table's name. * - * @return Indicate if table is delete successfully. + * @return Indicate if table is drop successfully. */ virtual Status DropTable(const std::string& table_name) = 0; @@ -239,9 +239,9 @@ class Connection { CreateIndex(const IndexParam& index_param) = 0; /** - * @brief Add vector to table + * @brief Insert vector to table * - * This method is used to add vector array to table. + * This method is used to insert vector array to table. * * @param table_name, target table's name. * @param partition_tag, target partition's tag, keep empty if no partition. @@ -264,6 +264,7 @@ class Connection { * @param query_record_array, all vector are going to be queried. * @param query_range_array, time ranges, if not specified, will search in whole table * @param topk, how many similarity vectors will be searched. + * @param nprobe, the number of centroids choose to search. * @param topk_query_result_array, result array. * * @return Indicate if query is successful. @@ -346,7 +347,7 @@ class Connection { * * This method is internal used. * - * @return Server status. + * @return Task information in tasktables. */ virtual std::string DumpTaskTables() const = 0; From a718c7c9bb51d99447213d7c4d15d12f860cc903 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Thu, 28 Nov 2019 19:08:15 +0800 Subject: [PATCH 2/3] Optimizer specify wrong gpu_id --- CHANGELOG.md | 1 + core/src/db/meta/MySQLMetaImpl.cpp | 3 ++- core/src/scheduler/optimizer/FaissFlatPass.cpp | 2 +- core/src/scheduler/optimizer/FaissIVFFlatPass.cpp | 2 +- core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp | 2 +- core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp | 2 +- core/src/sdk/include/MilvusApi.h | 4 ++-- 7 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fab5bf3bd0..ba7561c333 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#552 - Server down during building index_type: IVF_PQ using GPU-edition - \#561 - Milvus server should report exception/error message or terminate on mysql metadata backend error - \#599 - Build index log is incorrect +- \#602 - Optimizer specify wrong gpu_id ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 7b53e6361a..dcf3824fe1 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1639,7 +1639,8 @@ MySQLMetaImpl::FilesByType(const std::string& table_id, const std::vector& case (int)TableFileSchema::BACKUP: msg = msg + " backup files:" + std::to_string(backup_count); break; - default:break; + default: + break; } } ENGINE_LOG_DEBUG << msg; diff --git a/core/src/scheduler/optimizer/FaissFlatPass.cpp b/core/src/scheduler/optimizer/FaissFlatPass.cpp index 15ddc62f55..f7377d22ef 100644 --- a/core/src/scheduler/optimizer/FaissFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissFlatPass.cpp @@ -60,7 +60,7 @@ FaissFlatPass::Run(const TaskPtr& task) { auto best_device_id = count_ % gpus.size(); SERVER_LOG_DEBUG << "FaissFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id << " to search!"; count_++; - res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpus[best_device_id]); } auto label = std::make_shared(res_ptr); task->label() = label; diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp index 70909e8cd1..b8fe778432 100644 --- a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp @@ -63,7 +63,7 @@ FaissIVFFlatPass::Run(const TaskPtr& task) { SERVER_LOG_DEBUG << "FaissIVFFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id << " to search!"; count_++; - res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpus[best_device_id]); } auto label = std::make_shared(res_ptr); task->label() = label; diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index 7bcee6b07e..ad4184187d 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -61,7 +61,7 @@ FaissIVFSQ8HPass::Run(const TaskPtr& task) { SERVER_LOG_DEBUG << "FaissIVFSQ8HPass: nq > gpu_search_threshold, specify gpu" << best_device_id << " to search!"; count_++; - res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpus[best_device_id]); } auto label = std::make_shared(res_ptr); task->label() = label; diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp index 7df917000c..280b024894 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp @@ -63,7 +63,7 @@ FaissIVFSQ8Pass::Run(const TaskPtr& task) { SERVER_LOG_DEBUG << "FaissIVFSQ8Pass: nq > gpu_search_threshold, specify gpu" << best_device_id << " to search!"; count_++; - res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpus[best_device_id]); } auto label = std::make_shared(res_ptr); task->label() = label; diff --git a/core/src/sdk/include/MilvusApi.h b/core/src/sdk/include/MilvusApi.h index 22155db3be..52b88b71e9 100644 --- a/core/src/sdk/include/MilvusApi.h +++ b/core/src/sdk/include/MilvusApi.h @@ -85,8 +85,8 @@ struct RowRecord { * @brief TopK query result */ struct QueryResult { - std::vector ids; ///< Query ids result - std::vector distances; ///< Query distances result + std::vector ids; ///< Query ids result + std::vector distances; ///< Query distances result }; using TopKQueryResult = std::vector; ///< Topk query result From 018abff2ee2015f5163dc1385b04ce32c645e147 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Thu, 28 Nov 2019 19:38:22 +0800 Subject: [PATCH 3/3] [skip ci]Modify some annotation in MilvusApi.h --- core/src/sdk/include/MilvusApi.h | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/core/src/sdk/include/MilvusApi.h b/core/src/sdk/include/MilvusApi.h index 52b88b71e9..0937ce2f8e 100644 --- a/core/src/sdk/include/MilvusApi.h +++ b/core/src/sdk/include/MilvusApi.h @@ -142,8 +142,8 @@ class Connection { /** * @brief Connect * - * Connect function should be called before any operations - * Server will be connected after Connect return OK + * This method is used to connect server. + * Connect function should be called before any operations. * * @param param, use to provide server information * @@ -156,10 +156,10 @@ class Connection { /** * @brief Connect * - * Connect function should be called before any operations - * Server will be connected after Connect return OK + * This method is used to connect server. + * Connect function should be called before any operations. * - * @param uri, use to provide server information, example: milvus://ipaddress:port + * @param uri, use to provide server uri, example: milvus://ipaddress:port * * @return Indicate if connect is successful */ @@ -169,7 +169,7 @@ class Connection { /** * @brief connected * - * Connection status. + * This method is used to test whether server is connected. * * @return Indicate if connection status */ @@ -179,7 +179,7 @@ class Connection { /** * @brief Disconnect * - * Server will be disconnected after Disconnect return OK + * This method is used to disconnect server. * * @return Indicate if disconnect is successful */ @@ -189,7 +189,7 @@ class Connection { /** * @brief Create table method * - * This method is used to create table + * This method is used to create table. * * @param param, use to provide table information to be created. * @@ -201,7 +201,7 @@ class Connection { /** * @brief Test table existence method * - * This method is used to create table + * This method is used to create table. * * @param table_name, target table's name. * @@ -246,7 +246,10 @@ class Connection { * @param table_name, target table's name. * @param partition_tag, target partition's tag, keep empty if no partition. * @param record_array, vector array is inserted. - * @param id_array, after inserted every vector is given a id. + * @param id_array, + * specify id for each vector, + * if this array is empty, milvus will generate unique id for each vector, + * and return all ids by this parameter. * * @return Indicate if vector array are inserted successfully */ @@ -259,10 +262,10 @@ class Connection { * * This method is used to query vector in table. * - * @param table_name, target table's name, keep empty if no partition. - * @param partition_tags, target partitions. + * @param table_name, target table's name. + * @param partition_tags, target partitions, keep empty if no partition. * @param query_record_array, all vector are going to be queried. - * @param query_range_array, time ranges, if not specified, will search in whole table + * @param query_range_array, [deprecated] time ranges, if not specified, will search in whole table * @param topk, how many similarity vectors will be searched. * @param nprobe, the number of centroids choose to search. * @param topk_query_result_array, result array. @@ -305,7 +308,7 @@ class Connection { * * This method is used to list all tables. * - * @param table_array, all tables are push into the array. + * @param table_array, all tables in database. * * @return Indicate if this operation is successful. */ @@ -353,6 +356,7 @@ class Connection { DumpTaskTables() const = 0; /** + * [deprecated] * @brief delete tables by date range * * This method is used to delete table data by date range.