From 3cc783550aed2f79a1fd1a55723a376f823ac003 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 12 Jul 2019 12:19:32 +0800 Subject: [PATCH 1/2] MS-208 Add buildinde interface for C++ SDK Former-commit-id: a3fae74afd7ebf672d6eb6117c605f6f76e3f632 --- cpp/CHANGELOG.md | 1 + cpp/src/db/EngineFactory.cpp | 6 +- cpp/src/db/FaissExecutionEngine.cpp | 5 +- cpp/src/db/FaissExecutionEngine.h | 3 + .../sdk/examples/simple/src/ClientTest.cpp | 73 ++++++++++++++----- cpp/src/sdk/include/MilvusApi.h | 11 +++ cpp/src/sdk/src/client/ClientProxy.cpp | 16 ++++ cpp/src/sdk/src/client/ClientProxy.h | 2 + cpp/src/sdk/src/interface/ConnectionImpl.cpp | 5 ++ cpp/src/sdk/src/interface/ConnectionImpl.h | 2 + 10 files changed, 102 insertions(+), 22 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 845ec2ceda..6ea62b4efb 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -17,6 +17,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl - MS-204 - Support multi db_path - MS-206 - Support SQ8 index type +- MS-208 - Add buildinde interface for C++ SDK ## New Feature - MS-195 - Add nlist and use_blas_threshold conf diff --git a/cpp/src/db/EngineFactory.cpp b/cpp/src/db/EngineFactory.cpp index f45fb103b9..2ca4b24aa1 100644 --- a/cpp/src/db/EngineFactory.cpp +++ b/cpp/src/db/EngineFactory.cpp @@ -22,19 +22,19 @@ EngineFactory::Build(uint16_t dimension, switch (type) { case EngineType::FAISS_IDMAP: { execution_engine_ptr = - ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IDMap", "IDMap,Flat")); + ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IDMAP, "IDMap,Flat")); break; } case EngineType::FAISS_IVFFLAT: { execution_engine_ptr = - ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVF", "IDMap,Flat")); + ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IVF, "IDMap,Flat")); break; } case EngineType::FAISS_IVFSQ8: { execution_engine_ptr = - ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVFSQ8", "IDMap,Flat")); + ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IVFSQ8, "IDMap,Flat")); break; } diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index 26f144ff19..dd22f9cb07 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -162,7 +162,8 @@ Status FaissExecutionEngine::Cache() { Status FaissExecutionEngine::Init() { - if(build_index_type_ == "IVF") { + if(build_index_type_ == BUILD_INDEX_TYPE_IVF || + build_index_type_ == BUILD_INDEX_TYPE_IVFSQ8) { using namespace zilliz::milvus::server; ServerConfig &config = ServerConfig::GetInstance(); @@ -170,7 +171,7 @@ Status FaissExecutionEngine::Init() { nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000); nlist_ = engine_config.GetInt32Value(CONFIG_NLIST,16384); - } else if(build_index_type_ == "IDMap") { + } else if(build_index_type_ == BUILD_INDEX_TYPE_IDMAP) { ; } else { return Status::Error("Wrong index type: ", build_index_type_); diff --git a/cpp/src/db/FaissExecutionEngine.h b/cpp/src/db/FaissExecutionEngine.h index eda0e4d70e..7005061ff6 100644 --- a/cpp/src/db/FaissExecutionEngine.h +++ b/cpp/src/db/FaissExecutionEngine.h @@ -15,6 +15,9 @@ namespace zilliz { namespace milvus { namespace engine { +const static std::string BUILD_INDEX_TYPE_IDMAP = "IDMap"; +const static std::string BUILD_INDEX_TYPE_IVF = "IVF"; +const static std::string BUILD_INDEX_TYPE_IVFSQ8 = "IVFSQ8"; class FaissExecutionEngine : public ExecutionEngine { public: diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index f2513ac2c3..5b9e4f2bb1 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -8,6 +8,7 @@ #include #include +#include #include using namespace ::milvus; @@ -126,6 +127,43 @@ namespace { std::cout << "Waiting " << seconds << " seconds ..." << std::endl; sleep(seconds); } + + class TimeRecorder { + public: + TimeRecorder(const std::string& title) + : title_(title) { + start_ = std::chrono::system_clock::now(); + } + + ~TimeRecorder() { + std::chrono::system_clock::time_point end = std::chrono::system_clock::now(); + long span = (std::chrono::duration_cast (end - start_)).count(); + std::cout << title_ << " totally cost: " << span << " ms" << std::endl; + } + + private: + std::string title_; + std::chrono::system_clock::time_point start_; + }; + + void DoSearch(std::shared_ptr conn, + const std::vector& record_array, + const std::string& phase_name) { + std::vector query_range_array; + Range rg; + rg.start_value = CurrentTmDate(); + rg.end_value = CurrentTmDate(); + query_range_array.emplace_back(rg); + std::vector topk_query_result_array; + + { + TimeRecorder rc(phase_name); + Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array); + std::cout << "SearchVector function call status: " << stat.ToString() << std::endl; + } + + PrintSearchResult(topk_query_result_array); + } } void @@ -188,28 +226,29 @@ ClientTest::Test(const std::string& address, const std::string& port) { PrintRecordIdArray(record_ids); } - {//search vectors + std::vector search_record_array; + BuildVectors(SEARCH_TARGET, SEARCH_TARGET + NQ, search_record_array); + + {//search vectors without index Sleep(2); - - std::vector record_array; - BuildVectors(SEARCH_TARGET, SEARCH_TARGET + NQ, record_array); - - std::vector query_range_array; - Range rg; - rg.start_value = CurrentTmDate(); - rg.end_value = CurrentTmDate(); - query_range_array.emplace_back(rg); - std::vector topk_query_result_array; - Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array); - std::cout << "SearchVector function call status: " << stat.ToString() << std::endl; - PrintSearchResult(topk_query_result_array); + DoSearch(conn, search_record_array, "Search without index"); } - {//delete table - Status stat = conn->DeleteTable(TABLE_NAME); - std::cout << "DeleteTable function call status: " << stat.ToString() << std::endl; + {//wait unit build index finish + std::cout << "Wait until build all index done" << std::endl; + Status stat = conn->BuildIndex(TABLE_NAME); + std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl; } + {//search vectors after build index finish + DoSearch(conn, search_record_array, "Search after build index finish"); + } + +// {//delete table +// Status stat = conn->DeleteTable(TABLE_NAME); +// std::cout << "DeleteTable function call status: " << stat.ToString() << std::endl; +// } + {//server status std::string status = conn->ServerStatus(); std::cout << "Server status before disconnect: " << status << std::endl; diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index ec14621981..a7529d3e00 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -181,6 +181,17 @@ public: virtual Status DeleteTable(const std::string &table_name) = 0; + /** + * @brief Build index method + * + * This method is used to build index for whole table + * + * @param table_name, table name is going to be build index. + * + * @return Indicate if build index successfully. + */ + virtual Status BuildIndex(const std::string &table_name) = 0; + /** * @brief Add vector to table * diff --git a/cpp/src/sdk/src/client/ClientProxy.cpp b/cpp/src/sdk/src/client/ClientProxy.cpp index d26007cd0b..81dc8d28e4 100644 --- a/cpp/src/sdk/src/client/ClientProxy.cpp +++ b/cpp/src/sdk/src/client/ClientProxy.cpp @@ -126,6 +126,22 @@ ClientProxy::DeleteTable(const std::string &table_name) { return Status::OK(); } +Status +ClientProxy::BuildIndex(const std::string &table_name) { + if(!IsConnected()) { + return Status(StatusCode::NotConnected, "not connected to server"); + } + + try { + ClientPtr()->interface()->BuildIndex(table_name); + + } catch ( std::exception& ex) { + return Status(StatusCode::UnknownError, "failed to build index: " + std::string(ex.what())); + } + + return Status::OK(); +} + Status ClientProxy::AddVector(const std::string &table_name, const std::vector &record_array, diff --git a/cpp/src/sdk/src/client/ClientProxy.h b/cpp/src/sdk/src/client/ClientProxy.h index a2ede77c40..3e4850ef07 100644 --- a/cpp/src/sdk/src/client/ClientProxy.h +++ b/cpp/src/sdk/src/client/ClientProxy.h @@ -27,6 +27,8 @@ public: virtual Status DeleteTable(const std::string &table_name) override; + virtual Status BuildIndex(const std::string &table_name) override; + virtual Status AddVector(const std::string &table_name, const std::vector &record_array, std::vector &id_array) override; diff --git a/cpp/src/sdk/src/interface/ConnectionImpl.cpp b/cpp/src/sdk/src/interface/ConnectionImpl.cpp index efb5f61b1b..737cd7d83b 100644 --- a/cpp/src/sdk/src/interface/ConnectionImpl.cpp +++ b/cpp/src/sdk/src/interface/ConnectionImpl.cpp @@ -66,6 +66,11 @@ ConnectionImpl::DeleteTable(const std::string &table_name) { return client_proxy_->DeleteTable(table_name); } +Status +ConnectionImpl::BuildIndex(const std::string &table_name) { + return client_proxy_->BuildIndex(table_name); +} + Status ConnectionImpl::AddVector(const std::string &table_name, const std::vector &record_array, diff --git a/cpp/src/sdk/src/interface/ConnectionImpl.h b/cpp/src/sdk/src/interface/ConnectionImpl.h index 0f9cd14e39..60c6cd9b2d 100644 --- a/cpp/src/sdk/src/interface/ConnectionImpl.h +++ b/cpp/src/sdk/src/interface/ConnectionImpl.h @@ -29,6 +29,8 @@ public: virtual Status DeleteTable(const std::string &table_name) override; + virtual Status BuildIndex(const std::string &table_name) override; + virtual Status AddVector(const std::string &table_name, const std::vector &record_array, std::vector &id_array) override; From 48b306022bd9054d3c4e08491c1a43eeeb63e419 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 12 Jul 2019 12:21:37 +0800 Subject: [PATCH 2/2] typo Former-commit-id: 136fb068edc40479889c21ecc1f54506500b15ba --- cpp/src/sdk/examples/simple/src/ClientTest.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index 5b9e4f2bb1..b2f1c56ba4 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -244,10 +244,10 @@ ClientTest::Test(const std::string& address, const std::string& port) { DoSearch(conn, search_record_array, "Search after build index finish"); } -// {//delete table -// Status stat = conn->DeleteTable(TABLE_NAME); -// std::cout << "DeleteTable function call status: " << stat.ToString() << std::endl; -// } + {//delete table + Status stat = conn->DeleteTable(TABLE_NAME); + std::cout << "DeleteTable function call status: " << stat.ToString() << std::endl; + } {//server status std::string status = conn->ServerStatus();