admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

add id mapper

Browse Source 
Former-commit-id: 56cfee9e3beeb474e2a56c3e3a3fdda111cfabb2
This commit is contained in:
groot 2019-04-23 10:10:20 +08:00
parent 8909b19722
commit 3f7fe2548a
8 changed files with 130 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cpp/README.md
View File

@ -1,8 +1,11 @@
### Compilation
#### Step 1: install necessery tools
centos7 : yum install gfortran
ubuntu16.04 : sudo apt-install install gfortran libsqlite3-dev
centos7 :
yum install gfortran libsqlite3-dev libsnappy-dev zlib bzip2
ubuntu16.04 :
sudo apt-install install gfortran libsqlite3-dev libsnappy-dev zlib bzip2
#### Step 2: build third-parties
Note: If you want to debug into third-parties, you can build debug with CXXFLAGS='-g -O0' with option

12
cpp/src/CMakeLists.txt
View File

@ -24,6 +24,7 @@ set(vecwise_engine_src
${wrapper_files}
)
include_directories(/usr/include)
include_directories(/usr/local/cuda/include)
find_library(cuda_library cudart cublas HINTS /usr/local/cuda/lib64)
@ -37,6 +38,8 @@ set(engine_libs
sqlite3
)
link_libraries(/usr/lib/x86_64-linux-gnu)
target_link_libraries(vecwise_engine ${engine_libs} ${cuda_library})
add_executable(vecwise_server
@ -48,12 +51,17 @@ add_executable(vecwise_server
)
set(server_libs
vecwise_engine
rocksdb
thrift
pthread
yaml-cpp
boost_system
boost_filesystem
thrift
pthread
vecwise_engine
snappy
bz2
z
)
target_link_libraries(vecwise_server ${server_libs})

98
cpp/src/server/VecIdMapper.cpp
View File

@ -7,19 +7,25 @@
#include "VecIdMapper.h"
#include "ServerConfig.h"
#include "utils/Log.h"
#include "utils/CommonUtil.h"
#include "rocksdb/db.h"
#include "rocksdb/slice.h"
#include "rocksdb/options.h"
namespace zilliz {
namespace vecwise {
namespace server {
IVecIdMapper* IVecIdMapper::GetInstance() {
#if 1
#if 0
static SimpleIdMapper s_mapper;
return &s_mapper;
#else
ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER);
std::string db_path = config.GetValue(CONFIG_SERVER_DB_PATH);
db_path += "/id_mapping";
CommonUtil::CreateDirectory(db_path);
static RocksIdMapper s_mapper(db_path);
return &s_mapper;
#endif
@ -78,7 +84,7 @@ ServerError SimpleIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vect
auto iter = ids_.find(nid[i]);
if(iter == ids_.end()) {
sid.push_back("");
SERVER_LOG_ERROR << "Failed to find id: " << nid[i];
SERVER_LOG_ERROR << "ID mapper failed to find id: " << nid[i];
err = SERVER_INVALID_ARGUMENT;
continue;
}
@ -97,37 +103,117 @@ ServerError SimpleIdMapper::Delete(INTEGER_ID nid) {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
RocksIdMapper::RocksIdMapper(const std::string& store_path) {
rocksdb::Options options;
// Optimize RocksDB. This is the easiest way to get RocksDB to perform well
options.IncreaseParallelism();
options.OptimizeLevelStyleCompaction();
// create the DB if it's not already present
options.create_if_missing = true;
// open DB
rocksdb::Status s = rocksdb::DB::Open(options, store_path, &db_);
if(!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString();
db_ = nullptr;
}
}
RocksIdMapper::~RocksIdMapper() {
if(db_) {
db_->Close();
delete db_;
}
}
ServerError RocksIdMapper::Put(INTEGER_ID nid, const std::string& sid) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
std::string str_id = std::to_string(nid);//NOTE: keep a local virible here, since the Slice require a char* pointer
rocksdb::Slice key(str_id);
rocksdb::Slice value(sid);
rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), key, value);
if(!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString();
return SERVER_UNEXPECTED_ERROR;
}
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::Put(const std::vector<INTEGER_ID>& nid, const std::vector<std::string>& sid) {
return SERVER_SUCCESS;
return Put(nid.data(), nid.size(), sid);
}
ServerError RocksIdMapper::Put(const INTEGER_ID *nid, uint64_t count, const std::vector<std::string>& sid) {
if(count != sid.size()) {
return SERVER_INVALID_ARGUMENT;
}
ServerError err = SERVER_SUCCESS;
for(int64_t i = 0; i < count; i++) {
err = Put(nid[i], sid[i]);
if(err != SERVER_SUCCESS) {
return err;
}
}
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::Get(INTEGER_ID nid, std::string& sid) const {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
std::string str_id = std::to_string(nid);//NOTE: keep a local virible here, since the Slice require a char* pointer
rocksdb::Slice key(str_id);
rocksdb::Status s = db_->Get(rocksdb::ReadOptions(), key, &sid);
if(!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to get:" << s.ToString();
return SERVER_UNEXPECTED_ERROR;
}
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::Get(const std::vector<INTEGER_ID>& nid, std::vector<std::string>& sid) const {
return SERVER_SUCCESS;
return Get(nid.data(), nid.size(), sid);
}
ServerError RocksIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vector<std::string>& sid) const {
return SERVER_SUCCESS;
sid.clear();
ServerError err = SERVER_SUCCESS;
for(uint64_t i = 0; i < count; i++) {
std::string str_id;
ServerError temp_err = Get(nid[i], str_id);
if(temp_err != SERVER_SUCCESS) {
sid.push_back("");
SERVER_LOG_ERROR << "ID mapper failed to get id: " << nid[i];
err = temp_err;
continue;
}
sid.push_back(str_id);
}
return err;
}
ServerError RocksIdMapper::Delete(INTEGER_ID nid) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
std::string str_id = std::to_string(nid);//NOTE: keep a local virible here, since the Slice require a char* pointer
rocksdb::Slice key(str_id);
rocksdb::Status s = db_->Delete(rocksdb::WriteOptions(), key);
if(!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to delete:" << s.ToString();
return SERVER_UNEXPECTED_ERROR;
}
return SERVER_SUCCESS;
}

6
cpp/src/server/VecIdMapper.h
View File

@ -11,6 +11,10 @@
#include <vector>
#include <unordered_map>
namespace rocksdb {
class DB;
}
namespace zilliz {
namespace vecwise {
namespace server {
@ -68,6 +72,8 @@ public:
ServerError Delete(INTEGER_ID nid) override;
private:
rocksdb::DB* db_;
};
}

2
cpp/src/server/VecServiceHandler.cpp
View File

@ -23,6 +23,8 @@ VecServiceHandler::VecServiceHandler() {
std::string db_path = config.GetValue(CONFIG_SERVER_DB_PATH);
opt.meta.path = db_path + "/db";
CommonUtil::CreateDirectory(opt.meta.path);
zilliz::vecwise::engine::DB::Open(opt, &db_);
}

2
cpp/unittest/db/db_tests.cpp
View File

@ -34,7 +34,7 @@ TEST(DBTest, DB_TEST) {
group_info.dimension = group_dim;
group_info.group_id = group_name;
engine::Status stat = db->add_group(group_info);
ASSERT_STATS(stat);
//ASSERT_STATS(stat);
engine::meta::GroupSchema group_info_get;
group_info_get.group_id = group_name;

11
cpp/unittest/server/CMakeLists.txt
View File

@ -4,6 +4,7 @@
# Proprietary and confidential.
#-------------------------------------------------------------------------------
include_directories(../../src)
include_directories(/usr/include)
aux_source_directory(../../src/config config_files)
aux_source_directory(../../src/cache cache_srcs)
@ -12,6 +13,7 @@ aux_source_directory(./ test_srcs)
set(require_files
../../src/server/VecIdMapper.cpp
../../src/server/ServerConfig.cpp
../../src/utils/CommonUtil.cpp
)
add_executable(server_test
@ -26,7 +28,14 @@ set(require_libs
yaml-cpp
boost_system
boost_filesystem
pthread)
pthread
snappy
bz2
z
rocksdb
)
link_libraries(/usr/lib/x86_64-linux-gnu)
target_link_libraries(server_test
${unittest_libs}

6
cpp/unittest/server/idmapper_test.cpp
View File

@ -21,13 +21,15 @@ TEST(IdMapperTest, IDMAPPER_TEST) {
server::ServerError err = mapper->Put(nid, sid);
ASSERT_EQ(err, server::SERVER_SUCCESS);
sid.clear();
err = mapper->Put(nid, sid);
err = mapper->Put(nid, std::vector<std::string>());
ASSERT_NE(err, server::SERVER_SUCCESS);
std::vector<std::string> res;
err = mapper->Get(nid, res);
ASSERT_EQ(res.size(), nid.size());
for(size_t i = 0; i < res.size(); i++) {
ASSERT_EQ(res[i], sid[i]);
}
std::string str_id;
err = mapper->Get(50, str_id);