From c078cc85160924fead9600245043d2962e0c075d Mon Sep 17 00:00:00 2001 From: groot Date: Wed, 15 May 2019 19:57:31 +0800 Subject: [PATCH 1/9] avoid add vector to invalid group Former-commit-id: 3df82dd670ada8f24b3fb3a2411337e953b7c1b5 --- cpp/src/server/RocksIdMapper.cpp | 54 +++++++++++++++++++++++-------- cpp/src/server/RocksIdMapper.h | 5 ++- cpp/src/server/VecIdMapper.cpp | 11 +++++++ cpp/src/server/VecIdMapper.h | 6 ++++ cpp/src/server/VecServiceTask.cpp | 8 +++++ 5 files changed, 69 insertions(+), 15 deletions(-) diff --git a/cpp/src/server/RocksIdMapper.cpp b/cpp/src/server/RocksIdMapper.cpp index 2dba544243..1f24f6c9da 100644 --- a/cpp/src/server/RocksIdMapper.cpp +++ b/cpp/src/server/RocksIdMapper.cpp @@ -18,6 +18,8 @@ namespace zilliz { namespace vecwise { namespace server { +static const std::string ROCKSDB_DEFAULT_GROUP = "default"; + RocksIdMapper::RocksIdMapper() : db_(nullptr) { OpenDb(); @@ -90,6 +92,40 @@ void RocksIdMapper::CloseDb() { } } +//not thread-safe +ServerError RocksIdMapper::AddGroup(const std::string& group) { + if(!IsGroupExist(group)) { + if(db_ == nullptr) { + return SERVER_NULL_POINTER; + } + + try {//add group + rocksdb::ColumnFamilyHandle *cfh = nullptr; + rocksdb::Status s = db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), group, &cfh); + if (!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to create group:" << s.ToString(); + return SERVER_UNEXPECTED_ERROR; + } else { + column_handles_.insert(std::make_pair(group, cfh)); + } + } catch(std::exception& ex) { + SERVER_LOG_ERROR << "ID mapper failed to create group: " << ex.what(); + return SERVER_UNEXPECTED_ERROR; + } + } + + return SERVER_SUCCESS; +} + +//not thread-safe +bool RocksIdMapper::IsGroupExist(const std::string& group) const { + std::string group_name = group; + if(group_name.empty()){ + group_name = ROCKSDB_DEFAULT_GROUP; + } + return (column_handles_.count(group_name) > 0 && column_handles_[group_name] != nullptr); +} + ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { if(db_ == nullptr) { return SERVER_NULL_POINTER; @@ -104,22 +140,12 @@ ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, c return SERVER_UNEXPECTED_ERROR; } } else { - rocksdb::ColumnFamilyHandle *cfh = nullptr; - if(column_handles_.count(group) == 0) { - try {//add group - rocksdb::Status s = db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), group, &cfh); - if (!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to create group:" << s.ToString(); - } else { - column_handles_.insert(std::make_pair(group, cfh)); - } - } catch(std::exception& ex) { - std::cout << ex.what() << std::endl; - } - } else { - cfh = column_handles_[group]; + //try create group + if(AddGroup(group) != SERVER_SUCCESS){ + return SERVER_UNEXPECTED_ERROR; } + rocksdb::ColumnFamilyHandle *cfh = column_handles_[group]; rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), cfh, key, value); if (!s.ok()) { SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString(); diff --git a/cpp/src/server/RocksIdMapper.h b/cpp/src/server/RocksIdMapper.h index 8c73155903..70b77eee04 100644 --- a/cpp/src/server/RocksIdMapper.h +++ b/cpp/src/server/RocksIdMapper.h @@ -23,6 +23,9 @@ public: RocksIdMapper(); ~RocksIdMapper(); + ServerError AddGroup(const std::string& group) override; + bool IsGroupExist(const std::string& group) const override; + ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override; ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") override; @@ -38,7 +41,7 @@ private: private: rocksdb::DB* db_; - std::unordered_map column_handles_; + mutable std::unordered_map column_handles_; }; } diff --git a/cpp/src/server/VecIdMapper.cpp b/cpp/src/server/VecIdMapper.cpp index ecf5058a4b..b8bea7b348 100644 --- a/cpp/src/server/VecIdMapper.cpp +++ b/cpp/src/server/VecIdMapper.cpp @@ -39,6 +39,17 @@ SimpleIdMapper::~SimpleIdMapper() { } +ServerError SimpleIdMapper::AddGroup(const std::string& group) { + if(id_groups_.count(group) == 0) { + id_groups_.insert(std::make_pair(group, ID_MAPPING())); + } +} + +//not thread-safe +bool SimpleIdMapper::IsGroupExist(const std::string& group) const { + return id_groups_.count(group) > 0; +} + //not thread-safe ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { ID_MAPPING& mapping = id_groups_[group]; diff --git a/cpp/src/server/VecIdMapper.h b/cpp/src/server/VecIdMapper.h index 9bb6d500da..f3c2bdde27 100644 --- a/cpp/src/server/VecIdMapper.h +++ b/cpp/src/server/VecIdMapper.h @@ -25,6 +25,9 @@ public: virtual ~IVecIdMapper(){} + virtual ServerError AddGroup(const std::string& group) = 0; + virtual bool IsGroupExist(const std::string& group) const = 0; + virtual ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") = 0; virtual ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") = 0; @@ -41,6 +44,9 @@ public: SimpleIdMapper(); ~SimpleIdMapper(); + ServerError AddGroup(const std::string& group) override; + bool IsGroupExist(const std::string& group) const override; + ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override; ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") override; diff --git a/cpp/src/server/VecServiceTask.cpp b/cpp/src/server/VecServiceTask.cpp index 9ebe2c1bdf..9c996cb3a1 100644 --- a/cpp/src/server/VecServiceTask.cpp +++ b/cpp/src/server/VecServiceTask.cpp @@ -87,6 +87,7 @@ BaseTaskPtr AddGroupTask::Create(int32_t dimension, ServerError AddGroupTask::OnExecute() { try { + IVecIdMapper::GetInstance()->AddGroup(group_id_); engine::meta::GroupSchema group_info; group_info.dimension = (size_t)dimension_; group_info.group_id = group_id_; @@ -243,6 +244,13 @@ const AttribMap& AddVectorTask::GetVecAttrib() const { ServerError AddVectorTask::OnExecute() { try { + if(!IVecIdMapper::GetInstance()->IsGroupExist(group_id_)) { + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = "group not exist"; + SERVER_LOG_ERROR << error_msg_; + return error_code_; + } + uint64_t vec_dim = GetVecDimension(); std::vector vec_f; vec_f.resize(vec_dim); From 0d0c10df11a11ea12518eed6913541b8ef7ab8ed Mon Sep 17 00:00:00 2001 From: jinhai Date: Wed, 15 May 2019 20:22:04 +0800 Subject: [PATCH 2/9] Update license check Former-commit-id: ab88deb5aaeadda3626106a3a4cf3600131f528e --- cpp/CMakeLists.txt | 9 +++++---- cpp/src/license/LicenseCheck.cpp | 5 ++--- cpp/src/license/LicenseCheck.h | 1 - cpp/src/server/Server.cpp | 6 ++---- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c0eccae0ed..00a206484f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -45,14 +45,15 @@ endif () if(CMAKE_BUILD_TYPE STREQUAL "Release") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE") - if (GPU_VERSION STREQUAL "ON") - set(ENABLE_LICENSE "ON") - add_definitions("-DENABLE_LICENSE") - endif () else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE") endif() +if (GPU_VERSION STREQUAL "ON") + set(ENABLE_LICENSE "ON") + add_definitions("-DENABLE_LICENSE") +endif () + set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/" ${CMAKE_MODULE_PATH}) set(VECWISE_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include) diff --git a/cpp/src/license/LicenseCheck.cpp b/cpp/src/license/LicenseCheck.cpp index ca8fb4f930..4ea29663cf 100644 --- a/cpp/src/license/LicenseCheck.cpp +++ b/cpp/src/license/LicenseCheck.cpp @@ -74,7 +74,7 @@ LicenseCheck::AlterFile(const std::string &license_file_path, { exit(1); } - printf("---runing---\n"); +// printf("---runing---\n"); pt->expires_at(pt->expires_at() + boost::posix_time::hours(1)); pt->async_wait(boost::bind(AlterFile, license_file_path, boost::asio::placeholders::error, pt)); return SERVER_SUCCESS; @@ -83,8 +83,7 @@ LicenseCheck::AlterFile(const std::string &license_file_path, ServerError LicenseCheck::StartCountingDown(const std::string &license_file_path) { - - if (!LicenseLibrary::IsFileExistent(license_file_path)) return SERVER_LICENSE_FILE_NOT_EXIST; + if (!LicenseLibrary::IsFileExistent(license_file_path)) exit(1); boost::asio::io_service io; boost::asio::deadline_timer t(io, boost::posix_time::hours(1)); t.async_wait(boost::bind(AlterFile, license_file_path, boost::asio::placeholders::error, &t)); diff --git a/cpp/src/license/LicenseCheck.h b/cpp/src/license/LicenseCheck.h index 9a22f57f5a..0fd1d87f35 100644 --- a/cpp/src/license/LicenseCheck.h +++ b/cpp/src/license/LicenseCheck.h @@ -36,7 +36,6 @@ class LicenseCheck { static ServerError StartCountingDown(const std::string &license_file_path); - private: }; diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 8c79c5a215..3f51929baf 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -160,10 +160,8 @@ Server::Start() { exit(1); } - if(server::LicenseCheck::StartCountingDown(license_file_path) != SERVER_SUCCESS) { - SERVER_LOG_ERROR << "License counter start error"; - exit(1); - } + std::thread counting_down(&server::LicenseCheck::StartCountingDown, license_file_path); + counting_down.detach(); #endif // Handle Signal From c04a7c0e78b3e2873e48b9234644b39f43802ba2 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Thu, 16 May 2019 11:20:10 +0800 Subject: [PATCH 3/9] fix(db): fix create directory bug in release Former-commit-id: e060ff269f66c02c69978e0a7c77afae59528eb4 --- cpp/src/db/DBMetaImpl.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 3154615ed9..aaaaf21ce4 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -86,7 +86,11 @@ DBMetaImpl::DBMetaImpl(const DBMetaOptions& options_) Status DBMetaImpl::initialize() { if (!boost::filesystem::is_directory(_options.path)) { - assert(boost::filesystem::create_directory(_options.path)); + auto ret = boost::filesystem::create_directory(_options.path); + if (!ret) { + LOG(ERROR) << "Create directory " << _options.path << " Error"; + } + assert(ret); } ConnectorPtr = std::make_unique(StoragePrototype(_options.path+"/meta.sqlite")); @@ -123,7 +127,11 @@ Status DBMetaImpl::add_group(GroupSchema& group_info) { auto group_path = GetGroupPath(group_info.group_id); if (!boost::filesystem::is_directory(group_path)) { - assert(boost::filesystem::create_directory(group_path)); + auto ret = boost::filesystem::create_directory(group_path); + if (!ret) { + LOG(ERROR) << "Create directory " << group_path << " Error"; + } + assert(ret); } return Status::OK(); @@ -207,7 +215,11 @@ Status DBMetaImpl::add_group_file(GroupFileSchema& group_file) { auto partition_path = GetGroupDatePartitionPath(group_file.group_id, group_file.date); if (!boost::filesystem::is_directory(partition_path)) { - assert(boost::filesystem::create_directory(partition_path)); + auto ret = boost::filesystem::create_directory(partition_path); + if (!ret) { + LOG(ERROR) << "Create directory " << partition_path << " Error"; + } + assert(ret); } return Status::OK(); From 833f52c7a1f3b5750a48f003fd5aadad33507b04 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 17 May 2019 17:33:28 +0800 Subject: [PATCH 4/9] fix id miss bug Former-commit-id: ad0d56ad7e1faef38e31c773fcc8fd3f7c1fe664 --- cpp/src/server/VecServiceTask.cpp | 6 +++++- cpp/test_client/src/ClientTest.cpp | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cpp/src/server/VecServiceTask.cpp b/cpp/src/server/VecServiceTask.cpp index 9c996cb3a1..c2435ff51b 100644 --- a/cpp/src/server/VecServiceTask.cpp +++ b/cpp/src/server/VecServiceTask.cpp @@ -488,10 +488,14 @@ ServerError AddBatchVectorTask::OnExecute() { std::list> threads_list; uint64_t begin_index = 0, end_index = USE_MT; - while(end_index < vec_count) { + while(true) { threads_list.push_back( GetThreadPool().enqueue(&AddBatchVectorTask::ProcessIdMapping, this, vector_ids, begin_index, end_index, tensor_ids_)); + if(end_index >= vec_count) { + break; + } + begin_index = end_index; end_index += USE_MT; if(end_index > vec_count) { diff --git a/cpp/test_client/src/ClientTest.cpp b/cpp/test_client/src/ClientTest.cpp index 3a11e8bcc3..188f30a02e 100644 --- a/cpp/test_client/src/ClientTest.cpp +++ b/cpp/test_client/src/ClientTest.cpp @@ -211,7 +211,10 @@ TEST(AddVector, CLIENT_TEST) { server::TimeRecorder rc("Add " + std::to_string(count) + " binary vectors in one batch"); std::vector ids; session.interface()->add_binary_vector_batch(ids, group.id, bin_tensor_list_2); - ASSERT_TRUE(!ids.empty()); + ASSERT_EQ(ids.size(), bin_tensor_list_2.tensor_list.size()); + for(size_t i = 0; i < ids.size(); i++) { + ASSERT_TRUE(!ids[i].empty()); + } rc.Elapse("done!"); } #endif From 5f4e16f3d7c06a5bd244b8365943ba3b00a31e9a Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 19 May 2019 10:59:35 +0800 Subject: [PATCH 5/9] avoid potential multi-threads risk Former-commit-id: c99078422d17cc33ebb4c74e5e85ca0f4203a4b1 --- cpp/src/server/RocksIdMapper.cpp | 136 ++++++++++++++++++++----------- cpp/src/server/RocksIdMapper.h | 14 ++++ 2 files changed, 104 insertions(+), 46 deletions(-) diff --git a/cpp/src/server/RocksIdMapper.cpp b/cpp/src/server/RocksIdMapper.cpp index 1f24f6c9da..386058f00e 100644 --- a/cpp/src/server/RocksIdMapper.cpp +++ b/cpp/src/server/RocksIdMapper.cpp @@ -30,6 +30,8 @@ RocksIdMapper::~RocksIdMapper() { } void RocksIdMapper::OpenDb() { + std::lock_guard lck(db_mutex_); + if(db_) { return; } @@ -81,6 +83,8 @@ void RocksIdMapper::OpenDb() { } void RocksIdMapper::CloseDb() { + std::lock_guard lck(db_mutex_); + for(auto& iter : column_handles_) { delete iter.second; } @@ -92,9 +96,86 @@ void RocksIdMapper::CloseDb() { } } -//not thread-safe ServerError RocksIdMapper::AddGroup(const std::string& group) { - if(!IsGroupExist(group)) { + std::lock_guard lck(db_mutex_); + + return AddGroupInternal(group); +} + +bool RocksIdMapper::IsGroupExist(const std::string& group) const { + std::lock_guard lck(db_mutex_); + + return IsGroupExistInternal(group); +} + + +ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { + std::lock_guard lck(db_mutex_); + + return PutInternal(nid, sid, group); +} + +ServerError RocksIdMapper::Put(const std::vector& nid, const std::vector& sid, const std::string& group) { + if(nid.size() != sid.size()) { + return SERVER_INVALID_ARGUMENT; + } + + std::lock_guard lck(db_mutex_); + ServerError err = SERVER_SUCCESS; + for(size_t i = 0; i < nid.size(); i++) { + err = PutInternal(nid[i], sid[i], group); + if(err != SERVER_SUCCESS) { + return err; + } + } + + return err; +} + +ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const { + std::lock_guard lck(db_mutex_); + + return GetInternal(nid, sid, group); +} + +ServerError RocksIdMapper::Get(const std::vector& nid, std::vector& sid, const std::string& group) const { + sid.clear(); + + std::lock_guard lck(db_mutex_); + + ServerError err = SERVER_SUCCESS; + for(size_t i = 0; i < nid.size(); i++) { + std::string str_id; + ServerError temp_err = GetInternal(nid[i], str_id, group); + if(temp_err != SERVER_SUCCESS) { + sid.push_back(""); + SERVER_LOG_ERROR << "ID mapper failed to get id: " << nid[i]; + err = temp_err; + continue; + } + + sid.push_back(str_id); + } + + return err; +} + +ServerError RocksIdMapper::Delete(const std::string& nid, const std::string& group) { + std::lock_guard lck(db_mutex_); + + return DeleteInternal(nid, group); +} + +ServerError RocksIdMapper::DeleteGroup(const std::string& group) { + std::lock_guard lck(db_mutex_); + + return DeleteGroupInternal(group); +} + +//internal methods(whitout lock) +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +ServerError RocksIdMapper::AddGroupInternal(const std::string& group) { + if(!IsGroupExistInternal(group)) { if(db_ == nullptr) { return SERVER_NULL_POINTER; } @@ -117,8 +198,7 @@ ServerError RocksIdMapper::AddGroup(const std::string& group) { return SERVER_SUCCESS; } -//not thread-safe -bool RocksIdMapper::IsGroupExist(const std::string& group) const { +bool RocksIdMapper::IsGroupExistInternal(const std::string& group) const { std::string group_name = group; if(group_name.empty()){ group_name = ROCKSDB_DEFAULT_GROUP; @@ -126,7 +206,7 @@ bool RocksIdMapper::IsGroupExist(const std::string& group) const { return (column_handles_.count(group_name) > 0 && column_handles_[group_name] != nullptr); } -ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { +ServerError RocksIdMapper::PutInternal(const std::string& nid, const std::string& sid, const std::string& group) { if(db_ == nullptr) { return SERVER_NULL_POINTER; } @@ -141,7 +221,7 @@ ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, c } } else { //try create group - if(AddGroup(group) != SERVER_SUCCESS){ + if(AddGroupInternal(group) != SERVER_SUCCESS){ return SERVER_UNEXPECTED_ERROR; } @@ -156,23 +236,7 @@ ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, c return SERVER_SUCCESS; } -ServerError RocksIdMapper::Put(const std::vector& nid, const std::vector& sid, const std::string& group) { - if(nid.size() != sid.size()) { - return SERVER_INVALID_ARGUMENT; - } - - ServerError err = SERVER_SUCCESS; - for(size_t i = 0; i < nid.size(); i++) { - err = Put(nid[i], sid[i], group); - if(err != SERVER_SUCCESS) { - return err; - } - } - - return err; -} - -ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const { +ServerError RocksIdMapper::GetInternal(const std::string& nid, std::string& sid, const std::string& group) const { sid = ""; if(db_ == nullptr) { return SERVER_NULL_POINTER; @@ -199,28 +263,8 @@ ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid, const s return SERVER_SUCCESS; } -ServerError RocksIdMapper::Get(const std::vector& nid, std::vector& sid, const std::string& group) const { - sid.clear(); - - ServerError err = SERVER_SUCCESS; - for(size_t i = 0; i < nid.size(); i++) { - std::string str_id; - ServerError temp_err = Get(nid[i], str_id, group); - if(temp_err != SERVER_SUCCESS) { - sid.push_back(""); - SERVER_LOG_ERROR << "ID mapper failed to get id: " << nid[i]; - err = temp_err; - continue; - } - - sid.push_back(str_id); - } - - return err; -} - -ServerError RocksIdMapper::Delete(const std::string& nid, const std::string& group) { - if(db_ == nullptr) { +ServerError RocksIdMapper::DeleteInternal(const std::string& nid, const std::string& group) { + if(db_ == nullptr) { return SERVER_NULL_POINTER; } @@ -244,7 +288,7 @@ ServerError RocksIdMapper::Delete(const std::string& nid, const std::string& gro return SERVER_SUCCESS; } -ServerError RocksIdMapper::DeleteGroup(const std::string& group) { +ServerError RocksIdMapper::DeleteGroupInternal(const std::string& group) { if(db_ == nullptr) { return SERVER_NULL_POINTER; } diff --git a/cpp/src/server/RocksIdMapper.h b/cpp/src/server/RocksIdMapper.h index 70b77eee04..5fc4667e75 100644 --- a/cpp/src/server/RocksIdMapper.h +++ b/cpp/src/server/RocksIdMapper.h @@ -13,6 +13,7 @@ #include #include #include +#include namespace zilliz { namespace vecwise { @@ -39,9 +40,22 @@ private: void OpenDb(); void CloseDb(); + ServerError AddGroupInternal(const std::string& group); + + bool IsGroupExistInternal(const std::string& group) const; + + ServerError PutInternal(const std::string& nid, const std::string& sid, const std::string& group); + + ServerError GetInternal(const std::string& nid, std::string& sid, const std::string& group) const; + + ServerError DeleteInternal(const std::string& nid, const std::string& group); + + ServerError DeleteGroupInternal(const std::string& group); + private: rocksdb::DB* db_; mutable std::unordered_map column_handles_; + mutable std::mutex db_mutex_; }; } From 0c699ac84b3e2111700f067587fb02f3c7a56b22 Mon Sep 17 00:00:00 2001 From: groot Date: Thu, 23 May 2019 10:46:20 +0800 Subject: [PATCH 6/9] prepare for gpu index Former-commit-id: 048b5c0f4ae84a1afcb57d43c7763c7617d64d26 --- .gitignore | 4 ---- cpp/.gitignore | 6 ++++++ cpp/conf/server_config_template.yaml | 19 +++++++++++++++++++ cpp/src/CMakeLists.txt | 1 + 4 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 cpp/.gitignore create mode 100644 cpp/conf/server_config_template.yaml diff --git a/.gitignore b/.gitignore index d239dd72a6..c0b8dbb2d7 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,3 @@ cmake_build *.lo *.tar.gz *.log - -cpp/third_party/thrift-0.12.0/ -cpp/third_party/faiss-1.5.1 -cpp/megasearch/ diff --git a/cpp/.gitignore b/cpp/.gitignore new file mode 100644 index 0000000000..e99e0273f3 --- /dev/null +++ b/cpp/.gitignore @@ -0,0 +1,6 @@ +third_party/thrift-0.12.0/ +third_party/faiss-1.5.1/ +third_party/bzip2-1.0.6/ +third_party/sqlite3/ +megasearch/ +conf/server_config.yaml diff --git a/cpp/conf/server_config_template.yaml b/cpp/conf/server_config_template.yaml new file mode 100644 index 0000000000..fb6f6beae2 --- /dev/null +++ b/cpp/conf/server_config_template.yaml @@ -0,0 +1,19 @@ +server_config: + address: 0.0.0.0 + port: 33001 + transfer_protocol: json #optional: binary, compact, json, debug + server_mode: thread_pool #optional: simple, thread_pool + gpu_index: 0 #which gpu to be used + +db_config: + db_path: /tmp/vecwise + db_backend_url: http://127.0.0.1 + db_flush_interval: 5 #unit: second + idmapper_max_open_file: 128 + +license_config: + license_path: "/tmp/system.license" + +cache_config: + cpu_cache_capacity: 16 # unit: GB + gpu_cache_capacity: 2 # unit: GB \ No newline at end of file diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index d7978db37d..a3d9effb46 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -123,6 +123,7 @@ if (ENABLE_LICENSE STREQUAL "ON") add_executable(get_sys_info ${get_sys_info_src}) target_link_libraries(get_sys_info ${license_libs} vecwise_license) target_link_libraries(license_generator ${license_libs}) + install(TARGETS get_sys_info DESTINATION bin) endif () install(TARGETS vecwise_server DESTINATION bin) \ No newline at end of file From 9c954232311b7224d56cbdbc3d62231dbfccf0f4 Mon Sep 17 00:00:00 2001 From: groot Date: Thu, 23 May 2019 11:01:11 +0800 Subject: [PATCH 7/9] prepare for gpu index Former-commit-id: 764288e406f3a03822bca5e79b8d0ba9a3f4e13e --- cpp/conf/server_config.yaml | 35 ++++------------------------------- 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 40d10d0a5a..fb6f6beae2 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -1,45 +1,18 @@ server_config: address: 0.0.0.0 port: 33001 - transfer_protocol: json #optional: binary, compact, json, debug + transfer_protocol: json #optional: binary, compact, json, debug server_mode: thread_pool #optional: simple, thread_pool + gpu_index: 0 #which gpu to be used db_config: db_path: /tmp/vecwise db_backend_url: http://127.0.0.1 - db_flush_interval: 5 #unit: second + db_flush_interval: 5 #unit: second idmapper_max_open_file: 128 license_config: - license_path: "/home/jinhai/Documents/development/vecwise_engine/license/system.license" - -log_config: - global: - format: "%datetime | %level | %logger | %msg" - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log" - enabled: true - to_file: true - to_standard_output: true - subsecond_precision: 3 - performance_tracking: false - max_log_file_size: 2097152 # throw log files away after 2mb - debug: - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log" - enabled: true - warning: - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log" - trace: - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log" - verbose: - format: "%datetime{%d/%m/%y} | %level-%vlevel | %msg" - to_file: false - to_standard_output: true - error: - enabled: false - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log" - fatal: - enabled: false - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log" + license_path: "/tmp/system.license" cache_config: cpu_cache_capacity: 16 # unit: GB From 940e0d0593da8080e8b2359b41aac03265287bb6 Mon Sep 17 00:00:00 2001 From: jinhai Date: Sun, 26 May 2019 14:39:56 +0800 Subject: [PATCH 8/9] MS-4 Refactor the code structure Former-commit-id: 7113831bec73840f4cba826c9b63b69409653aed --- .gitignore | 14 +++++ cpp/CMakeLists.txt | 49 +++++++++++---- cpp/build.sh | 41 ++++++++----- cpp/cmake/Modules/ConfigureGoogleTest.cmake | 59 ------------------- .../Templates/GoogleTest.CMakeLists.txt.cmake | 12 ---- cpp/src/license/LicenseLibrary.cpp | 2 +- cpp/src/license/LicenseLibrary.h | 2 +- .../license/license_library_tests.cpp | 4 +- 8 files changed, 81 insertions(+), 102 deletions(-) delete mode 100644 cpp/cmake/Modules/ConfigureGoogleTest.cmake delete mode 100644 cpp/cmake/Templates/GoogleTest.CMakeLists.txt.cmake diff --git a/.gitignore b/.gitignore index c0b8dbb2d7..8600ed0dc7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,26 @@ +# CLion generated files +cpp/cmake-build-debug/ +cpp/cmake-build-release/ +cpp/cmake_build +cpp/.idea/ + .idea/ .ycm_extra_conf.py __pycache__ +# vscode generated files +.vscode + .env build cmake-build-debug cmake-build-release cmake_build + +# Compiled source +*.a +*.so +*.so.* *.o *.lo *.tar.gz diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 00a206484f..6fcb31a257 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -4,10 +4,32 @@ # Proprietary and confidential. #------------------------------------------------------------------------------- -cmake_minimum_required(VERSION 3.12) +cmake_minimum_required(VERSION 3.14) +message(STATUS "Building using CMake version: ${CMAKE_VERSION}") +set(MEGASEARCH_VERSION "0.1.0") + +string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" MEGASEARCH_BASE_VERSION "${MEGASEARCH_VERSION}") + +project(megasearch VERSION "${MEGASEARCH_BASE_VERSION}") project(vecwise_engine LANGUAGES CUDA CXX) +set(MEGASEARCH_VERSION_MAJOR "${megasearch_VERSION_MAJOR}") +set(MEGASEARCH_VERSION_MINOR "${megasearch_VERSION_MINOR}") +set(MEGASEARCH_VERSION_PATCH "${megasearch_VERSION_PATCH}") + +if(MEGASEARCH_VERSION_MAJOR STREQUAL "" + OR MEGASEARCH_VERSION_MINOR STREQUAL "" + OR MEGASEARCH_VERSION_PATCH STREQUAL "") + message(FATAL_ERROR "Failed to determine MegaSearch version from '${MEGASEARCH_VERSION}'") +endif() + +message(STATUS "MegaSearch version: " + "${MEGASEARCH_VERSION_MAJOR}.${MEGASEARCH_VERSION_MINOR}.${MEGASEARCH_VERSION_PATCH} " + "(full: '${MEGASEARCH_VERSION}')") + +set(MEGASEARCH_SOURCE_DIR ${PROJECT_SOURCE_DIR}) +set(MEGASEARCH_BINARY_DIR ${PROJECT_BINARY_DIR}) find_package(CUDA) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda") set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") @@ -32,16 +54,6 @@ else() set(VECWISE_BUILD_ARCH unknown) endif() -if(DEFINED UNIX) - message("building vecwise on Unix") - set(VECWISE_BUILD_SYSTEM macos) -elseif(DEFINED APPLE) - message("building vecwise on MacOS") - set(VECWISE_BUILD_SYSTEM unix) -else() - message("unknown OS") - set(VECWISE_BUILD_SYSTEM unknown) -endif () if(CMAKE_BUILD_TYPE STREQUAL "Release") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE") @@ -54,7 +66,19 @@ if (GPU_VERSION STREQUAL "ON") add_definitions("-DENABLE_LICENSE") endif () -set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/" ${CMAKE_MODULE_PATH}) +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +if (BUILD_UNIT_TEST) + option(MEGASEARCH_BUILD_TESTS "Build the megasearch test suite" ON) +endif(BUILD_UNIT_TEST) + +include(ExternalProject) +include(ThirdPartyPackages) + +include_directories(${MEGASEARCH_SOURCE_DIR}) +link_directories(${MEGASEARCH_BINARY_DIR}) + +## Following should be check set(VECWISE_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include) set(VECWISE_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) @@ -73,6 +97,7 @@ link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib64) #execute_process(COMMAND bash build.sh # WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third_party) + add_subdirectory(src) add_subdirectory(test_client) diff --git a/cpp/build.sh b/cpp/build.sh index f03b411bbb..fde83360ba 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -4,8 +4,9 @@ BUILD_TYPE="Debug" BUILD_UNITTEST="off" BUILD_GPU="OFF" INSTALL_PREFIX=$(pwd)/megasearch +MAKE_CLEAN="OFF" -while getopts "p:t:uhg" arg +while getopts "p:t:uhgr" arg do case $arg in t) @@ -21,6 +22,12 @@ do g) BUILD_GPU="ON" ;; + r) + if [[ -d cmake_build ]]; then + rm ./cmake_build -r + MAKE_CLEAN="ON" + fi + ;; h) # help echo " @@ -28,9 +35,11 @@ parameter: -t: build type -u: building unit test options -p: install prefix +-g: build GPU version +-r: remove previous build directory usage: -./build.sh -t \${BUILD_TYPE} [-u] [-h] +./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] " exit 0 ;; @@ -41,27 +50,29 @@ usage: esac done -if [[ -d cmake_build ]]; then - rm cmake_build -r +if [[ ! -d cmake_build ]]; then + mkdir cmake_build + MAKE_CLEAN="ON" fi -rm -rf ./cmake_build -mkdir cmake_build cd cmake_build CUDA_COMPILER=/usr/local/cuda/bin/nvcc -CMAKE_CMD="cmake -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ --DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} --DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ --DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ --DGPU_VERSION=${BUILD_GPU} \ -$@ ../" -echo ${CMAKE_CMD} +if [[ ${MAKE_CLEAN} = "ON" ]]; then + CMAKE_CMD="cmake -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ + -DGPU_VERSION=${BUILD_GPU} \ + $@ ../" + echo ${CMAKE_CMD} -${CMAKE_CMD} + ${CMAKE_CMD} + make clean +fi -make clean && make -j || exit 1 +make -j || exit 1 if [[ ${BUILD_TYPE} != "Debug" ]]; then strip src/vecwise_server diff --git a/cpp/cmake/Modules/ConfigureGoogleTest.cmake b/cpp/cmake/Modules/ConfigureGoogleTest.cmake deleted file mode 100644 index d2f2f2c666..0000000000 --- a/cpp/cmake/Modules/ConfigureGoogleTest.cmake +++ /dev/null @@ -1,59 +0,0 @@ -set(GTEST_ROOT "${CMAKE_BINARY_DIR}/googletest") - -set(GTEST_CMAKE_ARGS "") - # " -Dgtest_build_samples=ON" - # " -DCMAKE_VERBOSE_MAKEFILE=ON") - -if(NOT CMAKE_CXX11_ABI) - message(STATUS "GTEST: Disabling the GLIBCXX11 ABI") - list(APPEND GTEST_CMAKE_ARGS " -DCMAKE_C_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0") - list(APPEND GTEST_CMAKE_ARGS " -DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0") -elseif(CMAKE_CXX11_ABI) - message(STATUS "GTEST: Enabling the GLIBCXX11 ABI") - list(APPEND GTEST_CMAKE_ARGS " -DCMAKE_C_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=1") - list(APPEND GTEST_CMAKE_ARGS " -DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=1") -endif(NOT CMAKE_CXX11_ABI) - -configure_file("${CMAKE_SOURCE_DIR}/cmake/Templates/GoogleTest.CMakeLists.txt.cmake" - "${GTEST_ROOT}/CMakeLists.txt") - -file(MAKE_DIRECTORY "${GTEST_ROOT}/build") -file(MAKE_DIRECTORY "${GTEST_ROOT}/install") - -execute_process(COMMAND ${CMAKE_COMMAND} -G ${CMAKE_GENERATOR} . - RESULT_VARIABLE GTEST_CONFIG - WORKING_DIRECTORY ${GTEST_ROOT}) - -if(GTEST_CONFIG) - message(FATAL_ERROR "Configuring GoogleTest failed: " ${GTEST_CONFIG}) -endif(GTEST_CONFIG) - -set(PARALLEL_BUILD -j) -if($ENV{PARALLEL_LEVEL}) - set(NUM_JOBS $ENV{PARALLEL_LEVEL}) - set(PARALLEL_BUILD "${PARALLEL_BUILD}${NUM_JOBS}") -endif($ENV{PARALLEL_LEVEL}) - -if(${NUM_JOBS}) - if(${NUM_JOBS} EQUAL 1) - message(STATUS "GTEST BUILD: Enabling Sequential CMake build") - elseif(${NUM_JOBS} GREATER 1) - message(STATUS "GTEST BUILD: Enabling Parallel CMake build with ${NUM_JOBS} jobs") - endif(${NUM_JOBS} EQUAL 1) -else() - message(STATUS "GTEST BUILD: Enabling Parallel CMake build with all threads") -endif(${NUM_JOBS}) - -execute_process(COMMAND ${CMAKE_COMMAND} --build .. -- ${PARALLEL_BUILD} - RESULT_VARIABLE GTEST_BUILD - WORKING_DIRECTORY ${GTEST_ROOT}/build) - -if(GTEST_BUILD) - message(FATAL_ERROR "Building GoogleTest failed: " ${GTEST_BUILD}) -endif(GTEST_BUILD) - -message(STATUS "GoogleTest installed here: " ${GTEST_ROOT}/install) -set(GTEST_INCLUDE_DIR "${GTEST_ROOT}/install/include") -set(GTEST_LIBRARY_DIR "${GTEST_ROOT}/install/lib") -set(GTEST_FOUND TRUE) - diff --git a/cpp/cmake/Templates/GoogleTest.CMakeLists.txt.cmake b/cpp/cmake/Templates/GoogleTest.CMakeLists.txt.cmake deleted file mode 100644 index fe0035e240..0000000000 --- a/cpp/cmake/Templates/GoogleTest.CMakeLists.txt.cmake +++ /dev/null @@ -1,12 +0,0 @@ -cmake_minimum_required(VERSION 3.12) - -include(ExternalProject) - -ExternalProject_Add(GoogleTest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG release-1.8.0 - SOURCE_DIR "${GTEST_ROOT}/googletest" - BINARY_DIR "${GTEST_ROOT}/build" - INSTALL_DIR "${GTEST_ROOT}/install" - CMAKE_ARGS ${GTEST_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${GTEST_ROOT}/install) - diff --git a/cpp/src/license/LicenseLibrary.cpp b/cpp/src/license/LicenseLibrary.cpp index 669810552e..d67d7cdcd4 100644 --- a/cpp/src/license/LicenseLibrary.cpp +++ b/cpp/src/license/LicenseLibrary.cpp @@ -324,7 +324,7 @@ LicenseLibrary::GPUinfoFileDeserialization(const std::string &path, } ServerError -LicenseLibrary::GetDateTime(char *cha, time_t &data_time) { +LicenseLibrary::GetDateTime(const char *cha, time_t &data_time) { tm tm_; int year, month, day; sscanf(cha, "%d-%d-%d", &year, &month, &day); diff --git a/cpp/src/license/LicenseLibrary.h b/cpp/src/license/LicenseLibrary.h index a4202b1a0b..d5e97ac8a3 100644 --- a/cpp/src/license/LicenseLibrary.h +++ b/cpp/src/license/LicenseLibrary.h @@ -92,7 +92,7 @@ class LicenseLibrary { std::map &uuid_encrption_map); static ServerError - GetDateTime(char *cha, time_t &data_time); + GetDateTime(const char *cha, time_t &data_time); private: diff --git a/cpp/unittest/license/license_library_tests.cpp b/cpp/unittest/license/license_library_tests.cpp index f47ce5a43f..c68f243d4e 100644 --- a/cpp/unittest/license/license_library_tests.cpp +++ b/cpp/unittest/license/license_library_tests.cpp @@ -128,8 +128,8 @@ TEST(LicenseLibraryTest, LICENSE_FILE_TEST) { // 11.GetDateTime time_t starting_time; time_t end_time; - char *string_starting_time = "2019-05-10"; - char *string_end_time = "2022-05-10"; + const char *string_starting_time = "2019-05-10"; + const char *string_end_time = "2022-05-10"; err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time); ASSERT_EQ(err, server::SERVER_SUCCESS); err = server::LicenseLibrary::GetDateTime(string_end_time, end_time); From 9da40e0eacc96a36b1347f55fc9840dfd721a44b Mon Sep 17 00:00:00 2001 From: jinhai Date: Sun, 26 May 2019 14:42:09 +0800 Subject: [PATCH 9/9] MS-4 Refactor the code structure Former-commit-id: 96d0966168b3aaef9b12977ba9c65b0830cb44e4 --- CHANGELOGS.md | 0 CONTRIBUTING.md | 0 INSTALL.md | 0 LICENSE.md | 0 cpp/cmake/GoogleTest.cmake | 92 ++++++++++++++++++++++++++++++ cpp/cmake/ThirdPartyPackages.cmake | 66 +++++++++++++++++++++ docs/.gitignore | 0 python/.gitignore | 0 8 files changed, 158 insertions(+) create mode 100644 CHANGELOGS.md create mode 100644 CONTRIBUTING.md create mode 100644 INSTALL.md create mode 100644 LICENSE.md create mode 100644 cpp/cmake/GoogleTest.cmake create mode 100644 cpp/cmake/ThirdPartyPackages.cmake create mode 100644 docs/.gitignore create mode 100644 python/.gitignore diff --git a/CHANGELOGS.md b/CHANGELOGS.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cpp/cmake/GoogleTest.cmake b/cpp/cmake/GoogleTest.cmake new file mode 100644 index 0000000000..9ea62af1c4 --- /dev/null +++ b/cpp/cmake/GoogleTest.cmake @@ -0,0 +1,92 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Tries to find GTest headers and libraries. +# +# Usage of this module as follows: +# +# find_package(GTest) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# GTest_HOME - When set, this path is inspected instead of standard library +# locations as the root of the GTest installation. +# The environment variable GTEST_HOME overrides this veriable. +# +# This module defines +# GTEST_INCLUDE_DIR, directory containing headers +# GTEST_LIBS, directory containing gtest libraries +# GTEST_STATIC_LIB, path to libgtest.a +# GTEST_SHARED_LIB, path to libgtest's shared library +# GTEST_FOUND, whether gtest has been found + +if( NOT "${GTEST_HOME}" STREQUAL "") + file( TO_CMAKE_PATH "${GTEST_HOME}" _native_path ) + list( APPEND _gtest_roots ${_native_path} ) +elseif ( GTest_HOME ) + list( APPEND _gtest_roots ${GTest_HOME} ) +endif() + +# Try the parameterized roots, if they exist +if ( _gtest_roots ) + find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h + PATHS ${_gtest_roots} NO_DEFAULT_PATH + PATH_SUFFIXES "include" ) + find_library( GTEST_LIBRARIES NAMES gtest gtest_main + PATHS ${_gtest_roots} NO_DEFAULT_PATH + PATH_SUFFIXES "lib" ) +else () + find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h ) + find_library( GTEST_LIBRARIES NAMES gtest ) +endif () + + +if (GTEST_INCLUDE_DIR AND GTEST_LIBRARIES) + set(GTEST_FOUND TRUE) + get_filename_component( GTEST_LIBS ${GTEST_LIBRARIES} PATH ) + set(GTEST_LIB_NAME gtest) + set(GTEST_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}) + set(GTEST_MAIN_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}_main${CMAKE_STATIC_LIBRARY_SUFFIX}) + set(GTEST_SHARED_LIB ${GTEST_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) +else () + set(GTEST_FOUND FALSE) +endif () + +if (GTEST_FOUND) + if (NOT GTest_FIND_QUIETLY) + message(STATUS "Found the GTest library: ${GTEST_LIBRARIES}") + endif () +else () + if (NOT GTest_FIND_QUIETLY) + set(GTEST_ERR_MSG "Could not find the GTest library. Looked in ") + if ( _gtest_roots ) + set(GTEST_ERR_MSG "${GTEST_ERR_MSG} in ${_gtest_roots}.") + else () + set(GTEST_ERR_MSG "${GTEST_ERR_MSG} system search paths.") + endif () + if (GTest_FIND_REQUIRED) + message(FATAL_ERROR "${GTEST_ERR_MSG}") + else (GTest_FIND_REQUIRED) + message(STATUS "${GTEST_ERR_MSG}") + endif (GTest_FIND_REQUIRED) + endif () +endif () + +mark_as_advanced( + GTEST_INCLUDE_DIR + GTEST_LIBS + GTEST_LIBRARIES + GTEST_STATIC_LIB + GTEST_SHARED_LIB +) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake new file mode 100644 index 0000000000..e85025a427 --- /dev/null +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set(GTEST_VERSION "1.8.0") + +message(MEGASEARCH_BUILD_TESTS ${MEGASEARCH_BUILD_TESTS}) + +if(MEGASEARCH_BUILD_TESTS) + add_custom_target(unittest ctest -L unittest) + + if("$ENV{GTEST_HOME}" STREQUAL "") + message("Yes") + set(GTEST_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + + set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest/src/googletest") + set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") + set(GTEST_STATIC_LIB + "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GTEST_MAIN_STATIC_LIB + "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}") + + set(GTEST_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX} + -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}) + + ExternalProject_Add(googletest + URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz" + BUILD_BYPRODUCTS "${GTEST_STATIC_LIB}" "${GTEST_MAIN_STATIC_LIB}" + CMAKE_ARGS ${GTEST_CMAKE_ARGS} + ${EP_LOG_OPTIONS}) + set(GTEST_VENDORED 1) + else() + find_package(GTest REQUIRED) + set(GTEST_VENDORED 0) + endif() + + message(STATUS "GTest include dir: ${GTEST_INCLUDE_DIR}") + message(STATUS "GTest static library: ${GTEST_STATIC_LIB}") + include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) + + add_library(gtest STATIC IMPORTED) + set_target_properties(gtest PROPERTIES IMPORTED_LOCATION ${GTEST_STATIC_LIB}) + + add_library(gtest_main STATIC IMPORTED) + set_target_properties(gtest_main PROPERTIES IMPORTED_LOCATION + ${GTEST_MAIN_STATIC_LIB}) + + if(GTEST_VENDORED) + add_dependencies(gtest googletest) + add_dependencies(gtest_main googletest) + endif() +endif() \ No newline at end of file diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/.gitignore b/python/.gitignore new file mode 100644 index 0000000000..e69de29bb2