admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

MS-208 Add buildinde interface for C++ SDK

Browse Source 
Former-commit-id: a3fae74afd7ebf672d6eb6117c605f6f76e3f632
This commit is contained in:
starlord 2019-07-12 12:19:32 +08:00
parent e92f3f9274
commit 3cc783550a
10 changed files with 102 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/CHANGELOG.md
View File

@ -17,6 +17,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl
- MS-204 - Support multi db_path
- MS-206 - Support SQ8 index type
- MS-208 - Add buildinde interface for C++ SDK
## New Feature
- MS-195 - Add nlist and use_blas_threshold conf

6
cpp/src/db/EngineFactory.cpp
View File

@ -22,19 +22,19 @@ EngineFactory::Build(uint16_t dimension,
switch (type) {
case EngineType::FAISS_IDMAP: {
execution_engine_ptr =
ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IDMap", "IDMap,Flat"));
ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IDMAP, "IDMap,Flat"));
break;
}
case EngineType::FAISS_IVFFLAT: {
execution_engine_ptr =
ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVF", "IDMap,Flat"));
ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IVF, "IDMap,Flat"));
break;
}
case EngineType::FAISS_IVFSQ8: {
execution_engine_ptr =
ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVFSQ8", "IDMap,Flat"));
ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IVFSQ8, "IDMap,Flat"));
break;
}

5
cpp/src/db/FaissExecutionEngine.cpp
View File

@ -162,7 +162,8 @@ Status FaissExecutionEngine::Cache() {
Status FaissExecutionEngine::Init() {
if(build_index_type_ == "IVF") {
if(build_index_type_ == BUILD_INDEX_TYPE_IVF ||
build_index_type_ == BUILD_INDEX_TYPE_IVFSQ8) {
using namespace zilliz::milvus::server;
ServerConfig &config = ServerConfig::GetInstance();
@ -170,7 +171,7 @@ Status FaissExecutionEngine::Init() {
nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000);
nlist_ = engine_config.GetInt32Value(CONFIG_NLIST,16384);
} else if(build_index_type_ == "IDMap") {
} else if(build_index_type_ == BUILD_INDEX_TYPE_IDMAP) {
;
} else {
return Status::Error("Wrong index type: ", build_index_type_);

3
cpp/src/db/FaissExecutionEngine.h
View File

@ -15,6 +15,9 @@ namespace zilliz {
namespace milvus {
namespace engine {
const static std::string BUILD_INDEX_TYPE_IDMAP = "IDMap";
const static std::string BUILD_INDEX_TYPE_IVF = "IVF";
const static std::string BUILD_INDEX_TYPE_IVFSQ8 = "IVFSQ8";
class FaissExecutionEngine : public ExecutionEngine {
public:

73
cpp/src/sdk/examples/simple/src/ClientTest.cpp
View File

@ -8,6 +8,7 @@
#include <iostream>
#include <time.h>
#include <chrono>
#include <unistd.h>
using namespace ::milvus;
@ -126,6 +127,43 @@ namespace {
std::cout << "Waiting " << seconds << " seconds ..." << std::endl;
sleep(seconds);
}
class TimeRecorder {
public:
TimeRecorder(const std::string& title)
: title_(title) {
start_ = std::chrono::system_clock::now();
}
~TimeRecorder() {
std::chrono::system_clock::time_point end = std::chrono::system_clock::now();
long span = (std::chrono::duration_cast<std::chrono::milliseconds> (end - start_)).count();
std::cout << title_ << " totally cost: " << span << " ms" << std::endl;
}
private:
std::string title_;
std::chrono::system_clock::time_point start_;
};
void DoSearch(std::shared_ptr<Connection> conn,
const std::vector<RowRecord>& record_array,
const std::string& phase_name) {
std::vector<Range> query_range_array;
Range rg;
rg.start_value = CurrentTmDate();
rg.end_value = CurrentTmDate();
query_range_array.emplace_back(rg);
std::vector<TopKQueryResult> topk_query_result_array;
{
TimeRecorder rc(phase_name);
Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array);
std::cout << "SearchVector function call status: " << stat.ToString() << std::endl;
}
PrintSearchResult(topk_query_result_array);
}
}
void
@ -188,28 +226,29 @@ ClientTest::Test(const std::string& address, const std::string& port) {
PrintRecordIdArray(record_ids);
}
{//search vectors
std::vector<RowRecord> search_record_array;
BuildVectors(SEARCH_TARGET, SEARCH_TARGET + NQ, search_record_array);
{//search vectors without index
Sleep(2);
std::vector<RowRecord> record_array;
BuildVectors(SEARCH_TARGET, SEARCH_TARGET + NQ, record_array);
std::vector<Range> query_range_array;
Range rg;
rg.start_value = CurrentTmDate();
rg.end_value = CurrentTmDate();
query_range_array.emplace_back(rg);
std::vector<TopKQueryResult> topk_query_result_array;
Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array);
std::cout << "SearchVector function call status: " << stat.ToString() << std::endl;
PrintSearchResult(topk_query_result_array);
DoSearch(conn, search_record_array, "Search without index");
}
{//delete table
Status stat = conn->DeleteTable(TABLE_NAME);
std::cout << "DeleteTable function call status: " << stat.ToString() << std::endl;
{//wait unit build index finish
std::cout << "Wait until build all index done" << std::endl;
Status stat = conn->BuildIndex(TABLE_NAME);
std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl;
}
{//search vectors after build index finish
DoSearch(conn, search_record_array, "Search after build index finish");
}
// {//delete table
// Status stat = conn->DeleteTable(TABLE_NAME);
// std::cout << "DeleteTable function call status: " << stat.ToString() << std::endl;
// }
{//server status
std::string status = conn->ServerStatus();
std::cout << "Server status before disconnect: " << status << std::endl;

11
cpp/src/sdk/include/MilvusApi.h
View File

@ -181,6 +181,17 @@ public:
virtual Status DeleteTable(const std::string &table_name) = 0;
/**
* @brief Build index method
*
* This method is used to build index for whole table
*
* @param table_name, table name is going to be build index.
*
* @return Indicate if build index successfully.
*/
virtual Status BuildIndex(const std::string &table_name) = 0;
/**
* @brief Add vector to table
*

16
cpp/src/sdk/src/client/ClientProxy.cpp
View File

@ -126,6 +126,22 @@ ClientProxy::DeleteTable(const std::string &table_name) {
return Status::OK();
}
Status
ClientProxy::BuildIndex(const std::string &table_name) {
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
ClientPtr()->interface()->BuildIndex(table_name);
} catch ( std::exception& ex) {
return Status(StatusCode::UnknownError, "failed to build index: " + std::string(ex.what()));
}
return Status::OK();
}
Status
ClientProxy::AddVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,

2
cpp/src/sdk/src/client/ClientProxy.h
View File

@ -27,6 +27,8 @@ public:
virtual Status DeleteTable(const std::string &table_name) override;
virtual Status BuildIndex(const std::string &table_name) override;
virtual Status AddVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) override;

5
cpp/src/sdk/src/interface/ConnectionImpl.cpp
View File

@ -66,6 +66,11 @@ ConnectionImpl::DeleteTable(const std::string &table_name) {
return client_proxy_->DeleteTable(table_name);
}
Status
ConnectionImpl::BuildIndex(const std::string &table_name) {
return client_proxy_->BuildIndex(table_name);
}
Status
ConnectionImpl::AddVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,

2
cpp/src/sdk/src/interface/ConnectionImpl.h
View File

@ -29,6 +29,8 @@ public:
virtual Status DeleteTable(const std::string &table_name) override;
virtual Status BuildIndex(const std::string &table_name) override;
virtual Status AddVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) override;