From f642a7b38e854ca876c2ae6be0774923fa87e9f0 Mon Sep 17 00:00:00 2001 From: groot Date: Tue, 23 Apr 2019 16:34:43 +0800 Subject: [PATCH] redefine id mapper interface Former-commit-id: 6fa296aa09aae1019ed1e49a481c3a1e34440071 --- cpp/src/server/VecIdMapper.cpp | 59 +++++++++------------------ cpp/src/server/VecIdMapper.h | 41 ++++++++----------- cpp/src/server/VecServiceHandler.cpp | 20 +++++++-- cpp/unittest/server/idmapper_test.cpp | 13 +++--- 4 files changed, 59 insertions(+), 74 deletions(-) diff --git a/cpp/src/server/VecIdMapper.cpp b/cpp/src/server/VecIdMapper.cpp index aa7c11e5d7..a6bd1bbb14 100644 --- a/cpp/src/server/VecIdMapper.cpp +++ b/cpp/src/server/VecIdMapper.cpp @@ -40,28 +40,24 @@ SimpleIdMapper::~SimpleIdMapper() { } -ServerError SimpleIdMapper::Put(INTEGER_ID nid, const std::string& sid) { +ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid) { ids_[nid] = sid; return SERVER_SUCCESS; } -ServerError SimpleIdMapper::Put(const std::vector& nid, const std::vector& sid) { - return Put(nid.data(), nid.size(), sid); -} - -ServerError SimpleIdMapper::Put(const INTEGER_ID *nid, uint64_t count, const std::vector& sid) { - if(count != sid.size()) { +ServerError SimpleIdMapper::Put(const std::vector& nid, const std::vector& sid) { + if(nid.size() != sid.size()) { return SERVER_INVALID_ARGUMENT; } - for(int64_t i = 0; i < count; i++) { + for(size_t i = 0; i < nid.size(); i++) { ids_[nid[i]] = sid[i]; } return SERVER_SUCCESS; } -ServerError SimpleIdMapper::Get(INTEGER_ID nid, std::string& sid) const { +ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid) const { auto iter = ids_.find(nid); if(iter == ids_.end()) { return SERVER_INVALID_ARGUMENT; @@ -72,15 +68,11 @@ ServerError SimpleIdMapper::Get(INTEGER_ID nid, std::string& sid) const { return SERVER_SUCCESS; } -ServerError SimpleIdMapper::Get(const std::vector& nid, std::vector& sid) const { - return Get(nid.data(), nid.size(), sid); -} - -ServerError SimpleIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vector& sid) const { +ServerError SimpleIdMapper::Get(const std::vector& nid, std::vector& sid) const { sid.clear(); ServerError err = SERVER_SUCCESS; - for(uint64_t i = 0; i < count; i++) { + for(size_t i = 0; i < nid.size(); i++) { auto iter = ids_.find(nid[i]); if(iter == ids_.end()) { sid.push_back(""); @@ -95,7 +87,7 @@ ServerError SimpleIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vect return err; } -ServerError SimpleIdMapper::Delete(INTEGER_ID nid) { +ServerError SimpleIdMapper::Delete(const std::string& nid) { ids_.erase(nid); return SERVER_SUCCESS; } @@ -124,13 +116,12 @@ RocksIdMapper::~RocksIdMapper() { } } -ServerError RocksIdMapper::Put(INTEGER_ID nid, const std::string& sid) { +ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid) { if(db_ == nullptr) { return SERVER_NULL_POINTER; } - std::string str_id = std::to_string(nid);//NOTE: keep a local virible here, since the Slice require a char* pointer - rocksdb::Slice key(str_id); + rocksdb::Slice key(nid); rocksdb::Slice value(sid); rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), key, value); if(!s.ok()) { @@ -141,33 +132,28 @@ ServerError RocksIdMapper::Put(INTEGER_ID nid, const std::string& sid) { return SERVER_SUCCESS; } -ServerError RocksIdMapper::Put(const std::vector& nid, const std::vector& sid) { - return Put(nid.data(), nid.size(), sid); -} - -ServerError RocksIdMapper::Put(const INTEGER_ID *nid, uint64_t count, const std::vector& sid) { - if(count != sid.size()) { +ServerError RocksIdMapper::Put(const std::vector& nid, const std::vector& sid) { + if(nid.size() != sid.size()) { return SERVER_INVALID_ARGUMENT; } ServerError err = SERVER_SUCCESS; - for(int64_t i = 0; i < count; i++) { + for(size_t i = 0; i < nid.size(); i++) { err = Put(nid[i], sid[i]); if(err != SERVER_SUCCESS) { return err; } } - return SERVER_SUCCESS; + return err; } -ServerError RocksIdMapper::Get(INTEGER_ID nid, std::string& sid) const { +ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid) const { if(db_ == nullptr) { return SERVER_NULL_POINTER; } - std::string str_id = std::to_string(nid);//NOTE: keep a local virible here, since the Slice require a char* pointer - rocksdb::Slice key(str_id); + rocksdb::Slice key(nid); rocksdb::Status s = db_->Get(rocksdb::ReadOptions(), key, &sid); if(!s.ok()) { SERVER_LOG_ERROR << "ID mapper failed to get:" << s.ToString(); @@ -177,15 +163,11 @@ ServerError RocksIdMapper::Get(INTEGER_ID nid, std::string& sid) const { return SERVER_SUCCESS; } -ServerError RocksIdMapper::Get(const std::vector& nid, std::vector& sid) const { - return Get(nid.data(), nid.size(), sid); -} - -ServerError RocksIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vector& sid) const { +ServerError RocksIdMapper::Get(const std::vector& nid, std::vector& sid) const { sid.clear(); ServerError err = SERVER_SUCCESS; - for(uint64_t i = 0; i < count; i++) { + for(size_t i = 0; i < nid.size(); i++) { std::string str_id; ServerError temp_err = Get(nid[i], str_id); if(temp_err != SERVER_SUCCESS) { @@ -201,13 +183,12 @@ ServerError RocksIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vecto return err; } -ServerError RocksIdMapper::Delete(INTEGER_ID nid) { +ServerError RocksIdMapper::Delete(const std::string& nid) { if(db_ == nullptr) { return SERVER_NULL_POINTER; } - std::string str_id = std::to_string(nid);//NOTE: keep a local virible here, since the Slice require a char* pointer - rocksdb::Slice key(str_id); + rocksdb::Slice key(nid); rocksdb::Status s = db_->Delete(rocksdb::WriteOptions(), key); if(!s.ok()) { SERVER_LOG_ERROR << "ID mapper failed to delete:" << s.ToString(); diff --git a/cpp/src/server/VecIdMapper.h b/cpp/src/server/VecIdMapper.h index 4a24df0099..c1748980b7 100644 --- a/cpp/src/server/VecIdMapper.h +++ b/cpp/src/server/VecIdMapper.h @@ -19,23 +19,20 @@ namespace zilliz { namespace vecwise { namespace server { -using INTEGER_ID = int64_t; - class IVecIdMapper { public: static IVecIdMapper* GetInstance(); virtual ~IVecIdMapper(){} - virtual ServerError Put(INTEGER_ID nid, const std::string& sid) = 0; - virtual ServerError Put(const std::vector& nid, const std::vector& sid) = 0; - virtual ServerError Put(const INTEGER_ID *nid, uint64_t count, const std::vector& sid) = 0; + virtual ServerError Put(const std::string& nid, const std::string& sid) = 0; + virtual ServerError Put(const std::vector& nid, const std::vector& sid) = 0; - virtual ServerError Get(INTEGER_ID nid, std::string& sid) const = 0; - virtual ServerError Get(const std::vector& nid, std::vector& sid) const = 0; - virtual ServerError Get(const INTEGER_ID *nid, uint64_t count, std::vector& sid) const = 0; + virtual ServerError Get(const std::string& nid, std::string& sid) const = 0; + //NOTE: the 'sid' will be cleared at begin of the function + virtual ServerError Get(const std::vector& nid, std::vector& sid) const = 0; - virtual ServerError Delete(INTEGER_ID nid) = 0; + virtual ServerError Delete(const std::string& nid) = 0; }; class SimpleIdMapper : public IVecIdMapper{ @@ -43,18 +40,16 @@ public: SimpleIdMapper(); ~SimpleIdMapper(); - ServerError Put(INTEGER_ID nid, const std::string& sid) override; - ServerError Put(const std::vector& nid, const std::vector& sid) override; - ServerError Put(const INTEGER_ID *nid, uint64_t count, const std::vector& sid) override; + ServerError Put(const std::string& nid, const std::string& sid) override; + ServerError Put(const std::vector& nid, const std::vector& sid) override; - ServerError Get(INTEGER_ID nid, std::string& sid) const override; - ServerError Get(const std::vector& nid, std::vector& sid) const override; - ServerError Get(const INTEGER_ID *nid, uint64_t count, std::vector& sid) const override; + ServerError Get(const std::string& nid, std::string& sid) const override; + ServerError Get(const std::vector& nid, std::vector& sid) const override; - ServerError Delete(INTEGER_ID nid) override; + ServerError Delete(const std::string& nid) override; private: - std::unordered_map ids_; + std::unordered_map ids_; }; class RocksIdMapper : public IVecIdMapper{ @@ -62,15 +57,13 @@ public: RocksIdMapper(const std::string& store_path); ~RocksIdMapper(); - ServerError Put(INTEGER_ID nid, const std::string& sid) override; - ServerError Put(const std::vector& nid, const std::vector& sid) override; - ServerError Put(const INTEGER_ID *nid, uint64_t count, const std::vector& sid) override; + ServerError Put(const std::string& nid, const std::string& sid) override; + ServerError Put(const std::vector& nid, const std::vector& sid) override; - ServerError Get(INTEGER_ID nid, std::string& sid) const override; - ServerError Get(const std::vector& nid, std::vector& sid) const override; - ServerError Get(const INTEGER_ID *nid, uint64_t count, std::vector& sid) const override; + ServerError Get(const std::string& nid, std::string& sid) const override; + ServerError Get(const std::vector& nid, std::vector& sid) const override; - ServerError Delete(INTEGER_ID nid) override; + ServerError Delete(const std::string& nid) override; private: rocksdb::DB* db_; diff --git a/cpp/src/server/VecServiceHandler.cpp b/cpp/src/server/VecServiceHandler.cpp index 5465412877..8d12ef5962 100644 --- a/cpp/src/server/VecServiceHandler.cpp +++ b/cpp/src/server/VecServiceHandler.cpp @@ -100,7 +100,8 @@ VecServiceHandler::add_vector(const std::string &group_id, const VecTensor &tens if(vector_ids.size() != 1) { SERVER_LOG_ERROR << "Vector ID not returned"; } else { - IVecIdMapper::GetInstance()->Put(vector_ids[0], tensor.uid); + std::string nid = group_id + "_" + std::to_string(vector_ids[0]); + IVecIdMapper::GetInstance()->Put(nid, tensor.uid); } } @@ -131,8 +132,10 @@ VecServiceHandler::add_vector_batch(const std::string &group_id, if(vector_ids.size() != tensor_list.tensor_list.size()) { SERVER_LOG_ERROR << "Vector ID not returned"; } else { + std::string nid_prefix = group_id + "_"; for(size_t i = 0; i < vector_ids.size(); i++) { - IVecIdMapper::GetInstance()->Put(vector_ids[i], tensor_list.tensor_list[i].uid); + std::string nid = nid_prefix + std::to_string(vector_ids[i]); + IVecIdMapper::GetInstance()->Put(nid, tensor_list.tensor_list[i].uid); } } } @@ -163,9 +166,11 @@ VecServiceHandler::search_vector(VecSearchResult &_return, SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); } else { if(!results.empty()) { + std::string nid_prefix = group_id + "_"; for(auto id : results[0]) { std::string sid; - IVecIdMapper::GetInstance()->Get(id, sid); + std::string nid = nid_prefix + std::to_string(id); + IVecIdMapper::GetInstance()->Get(nid, sid); _return.id_list.push_back(sid); } } @@ -201,7 +206,14 @@ VecServiceHandler::search_vector_batch(VecSearchResultList &_return, } else { for(engine::QueryResult& res : results){ VecSearchResult v_res; - IVecIdMapper::GetInstance()->Get(res.data(), res.size(), v_res.id_list); + std::string nid_prefix = group_id + "_"; + for(auto id : results[0]) { + std::string sid; + std::string nid = nid_prefix + std::to_string(id); + IVecIdMapper::GetInstance()->Get(nid, sid); + v_res.id_list.push_back(sid); + } + _return.result_list.push_back(v_res); } } diff --git a/cpp/unittest/server/idmapper_test.cpp b/cpp/unittest/server/idmapper_test.cpp index 2563761a0c..76602e59ee 100644 --- a/cpp/unittest/server/idmapper_test.cpp +++ b/cpp/unittest/server/idmapper_test.cpp @@ -19,7 +19,7 @@ TEST(IdMapperTest, IDMAPPER_TEST) { server::IVecIdMapper* mapper = server::IVecIdMapper::GetInstance(); - std::vector nid = {1,50, 900, 10000}; + std::vector nid = {"1", "50", "900", "10000"}; std::vector sid = {"one", "fifty", "nine zero zero", "many"}; server::ServerError err = mapper->Put(nid, sid); ASSERT_EQ(err, server::SERVER_SUCCESS); @@ -35,22 +35,21 @@ TEST(IdMapperTest, IDMAPPER_TEST) { } std::string str_id; - err = mapper->Get(50, str_id); + err = mapper->Get(nid[1], str_id); ASSERT_EQ(str_id, "fifty"); - err = mapper->Delete(900); + err = mapper->Delete(nid[2]); ASSERT_EQ(err, server::SERVER_SUCCESS); - err = mapper->Get(900, str_id); + err = mapper->Get(nid[2], str_id); ASSERT_NE(err, server::SERVER_SUCCESS); - - //performance? + //test performance nid.clear(); sid.clear(); const int64_t count = 1000000; for(int64_t i = 0; i < count; i++) { - nid.push_back(i+100000); + nid.push_back(std::to_string(i+100000)); sid.push_back("val_" + std::to_string(i)); }