diff --git a/cpp/README.md b/cpp/README.md index 57f8043233..e8fd0af659 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -1,8 +1,11 @@ ### Compilation #### Step 1: install necessery tools - centos7 : yum install gfortran - ubuntu16.04 : sudo apt-install install gfortran libsqlite3-dev + centos7 : + yum install gfortran libsqlite3-dev libsnappy-dev zlib bzip2 + + ubuntu16.04 : + sudo apt-install install gfortran libsqlite3-dev libsnappy-dev zlib bzip2 #### Step 2: build third-parties Note: If you want to debug into third-parties, you can build debug with CXXFLAGS='-g -O0' with option diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 7cf96014cc..772c042e66 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -24,6 +24,7 @@ set(vecwise_engine_src ${wrapper_files} ) +include_directories(/usr/include) include_directories(/usr/local/cuda/include) find_library(cuda_library cudart cublas HINTS /usr/local/cuda/lib64) @@ -37,6 +38,8 @@ set(engine_libs sqlite3 ) +link_libraries(/usr/lib/x86_64-linux-gnu) + target_link_libraries(vecwise_engine ${engine_libs} ${cuda_library}) add_executable(vecwise_server @@ -48,12 +51,17 @@ add_executable(vecwise_server ) set(server_libs + vecwise_engine + rocksdb + thrift + pthread yaml-cpp boost_system boost_filesystem - thrift pthread - vecwise_engine + snappy + bz2 + z ) target_link_libraries(vecwise_server ${server_libs}) diff --git a/cpp/src/server/VecIdMapper.cpp b/cpp/src/server/VecIdMapper.cpp index abe3a49e1f..aa7c11e5d7 100644 --- a/cpp/src/server/VecIdMapper.cpp +++ b/cpp/src/server/VecIdMapper.cpp @@ -7,19 +7,25 @@ #include "VecIdMapper.h" #include "ServerConfig.h" #include "utils/Log.h" +#include "utils/CommonUtil.h" + +#include "rocksdb/db.h" +#include "rocksdb/slice.h" +#include "rocksdb/options.h" namespace zilliz { namespace vecwise { namespace server { IVecIdMapper* IVecIdMapper::GetInstance() { -#if 1 +#if 0 static SimpleIdMapper s_mapper; return &s_mapper; #else ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); std::string db_path = config.GetValue(CONFIG_SERVER_DB_PATH); db_path += "/id_mapping"; + CommonUtil::CreateDirectory(db_path); static RocksIdMapper s_mapper(db_path); return &s_mapper; #endif @@ -78,7 +84,7 @@ ServerError SimpleIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vect auto iter = ids_.find(nid[i]); if(iter == ids_.end()) { sid.push_back(""); - SERVER_LOG_ERROR << "Failed to find id: " << nid[i]; + SERVER_LOG_ERROR << "ID mapper failed to find id: " << nid[i]; err = SERVER_INVALID_ARGUMENT; continue; } @@ -97,37 +103,117 @@ ServerError SimpleIdMapper::Delete(INTEGER_ID nid) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// RocksIdMapper::RocksIdMapper(const std::string& store_path) { + rocksdb::Options options; + // Optimize RocksDB. This is the easiest way to get RocksDB to perform well + options.IncreaseParallelism(); + options.OptimizeLevelStyleCompaction(); + // create the DB if it's not already present + options.create_if_missing = true; + // open DB + rocksdb::Status s = rocksdb::DB::Open(options, store_path, &db_); + if(!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); + db_ = nullptr; + } } RocksIdMapper::~RocksIdMapper() { - + if(db_) { + db_->Close(); + delete db_; + } } ServerError RocksIdMapper::Put(INTEGER_ID nid, const std::string& sid) { + if(db_ == nullptr) { + return SERVER_NULL_POINTER; + } + + std::string str_id = std::to_string(nid);//NOTE: keep a local virible here, since the Slice require a char* pointer + rocksdb::Slice key(str_id); + rocksdb::Slice value(sid); + rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), key, value); + if(!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString(); + return SERVER_UNEXPECTED_ERROR; + } + return SERVER_SUCCESS; } ServerError RocksIdMapper::Put(const std::vector& nid, const std::vector& sid) { - return SERVER_SUCCESS; + return Put(nid.data(), nid.size(), sid); } ServerError RocksIdMapper::Put(const INTEGER_ID *nid, uint64_t count, const std::vector& sid) { + if(count != sid.size()) { + return SERVER_INVALID_ARGUMENT; + } + + ServerError err = SERVER_SUCCESS; + for(int64_t i = 0; i < count; i++) { + err = Put(nid[i], sid[i]); + if(err != SERVER_SUCCESS) { + return err; + } + } + return SERVER_SUCCESS; } ServerError RocksIdMapper::Get(INTEGER_ID nid, std::string& sid) const { + if(db_ == nullptr) { + return SERVER_NULL_POINTER; + } + + std::string str_id = std::to_string(nid);//NOTE: keep a local virible here, since the Slice require a char* pointer + rocksdb::Slice key(str_id); + rocksdb::Status s = db_->Get(rocksdb::ReadOptions(), key, &sid); + if(!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to get:" << s.ToString(); + return SERVER_UNEXPECTED_ERROR; + } + return SERVER_SUCCESS; } ServerError RocksIdMapper::Get(const std::vector& nid, std::vector& sid) const { - return SERVER_SUCCESS; + return Get(nid.data(), nid.size(), sid); } ServerError RocksIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vector& sid) const { - return SERVER_SUCCESS; + sid.clear(); + + ServerError err = SERVER_SUCCESS; + for(uint64_t i = 0; i < count; i++) { + std::string str_id; + ServerError temp_err = Get(nid[i], str_id); + if(temp_err != SERVER_SUCCESS) { + sid.push_back(""); + SERVER_LOG_ERROR << "ID mapper failed to get id: " << nid[i]; + err = temp_err; + continue; + } + + sid.push_back(str_id); + } + + return err; } ServerError RocksIdMapper::Delete(INTEGER_ID nid) { + if(db_ == nullptr) { + return SERVER_NULL_POINTER; + } + + std::string str_id = std::to_string(nid);//NOTE: keep a local virible here, since the Slice require a char* pointer + rocksdb::Slice key(str_id); + rocksdb::Status s = db_->Delete(rocksdb::WriteOptions(), key); + if(!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to delete:" << s.ToString(); + return SERVER_UNEXPECTED_ERROR; + } + return SERVER_SUCCESS; } diff --git a/cpp/src/server/VecIdMapper.h b/cpp/src/server/VecIdMapper.h index 04a484842e..4a24df0099 100644 --- a/cpp/src/server/VecIdMapper.h +++ b/cpp/src/server/VecIdMapper.h @@ -11,6 +11,10 @@ #include #include +namespace rocksdb { + class DB; +} + namespace zilliz { namespace vecwise { namespace server { @@ -68,6 +72,8 @@ public: ServerError Delete(INTEGER_ID nid) override; +private: + rocksdb::DB* db_; }; } diff --git a/cpp/src/server/VecServiceHandler.cpp b/cpp/src/server/VecServiceHandler.cpp index 0cb0069eac..2df7c5cb3a 100644 --- a/cpp/src/server/VecServiceHandler.cpp +++ b/cpp/src/server/VecServiceHandler.cpp @@ -23,6 +23,8 @@ VecServiceHandler::VecServiceHandler() { std::string db_path = config.GetValue(CONFIG_SERVER_DB_PATH); opt.meta.path = db_path + "/db"; + CommonUtil::CreateDirectory(opt.meta.path); + zilliz::vecwise::engine::DB::Open(opt, &db_); } diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 22513cc736..6dd1751a69 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -34,7 +34,7 @@ TEST(DBTest, DB_TEST) { group_info.dimension = group_dim; group_info.group_id = group_name; engine::Status stat = db->add_group(group_info); - ASSERT_STATS(stat); + //ASSERT_STATS(stat); engine::meta::GroupSchema group_info_get; group_info_get.group_id = group_name; diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 5bfb5e7b5d..0aee4e7ff2 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -4,6 +4,7 @@ # Proprietary and confidential. #------------------------------------------------------------------------------- include_directories(../../src) +include_directories(/usr/include) aux_source_directory(../../src/config config_files) aux_source_directory(../../src/cache cache_srcs) @@ -12,6 +13,7 @@ aux_source_directory(./ test_srcs) set(require_files ../../src/server/VecIdMapper.cpp ../../src/server/ServerConfig.cpp + ../../src/utils/CommonUtil.cpp ) add_executable(server_test @@ -26,7 +28,14 @@ set(require_libs yaml-cpp boost_system boost_filesystem - pthread) + pthread + snappy + bz2 + z + rocksdb + ) + +link_libraries(/usr/lib/x86_64-linux-gnu) target_link_libraries(server_test ${unittest_libs} diff --git a/cpp/unittest/server/idmapper_test.cpp b/cpp/unittest/server/idmapper_test.cpp index bd894a17c7..d9348212df 100644 --- a/cpp/unittest/server/idmapper_test.cpp +++ b/cpp/unittest/server/idmapper_test.cpp @@ -21,13 +21,15 @@ TEST(IdMapperTest, IDMAPPER_TEST) { server::ServerError err = mapper->Put(nid, sid); ASSERT_EQ(err, server::SERVER_SUCCESS); - sid.clear(); - err = mapper->Put(nid, sid); + err = mapper->Put(nid, std::vector()); ASSERT_NE(err, server::SERVER_SUCCESS); std::vector res; err = mapper->Get(nid, res); ASSERT_EQ(res.size(), nid.size()); + for(size_t i = 0; i < res.size(); i++) { + ASSERT_EQ(res[i], sid[i]); + } std::string str_id; err = mapper->Get(50, str_id);