From e2bf433de9259a26726b0f29dce04d21bc41e44f Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 15 Apr 2019 10:24:48 +0800 Subject: [PATCH 01/34] add thrift to third-party Former-commit-id: 8a3359ee295e4b1b65757917fe080460f7d37b7d From 36268f84ddabfc94c8d01c38b8d60fb3abf5bd27 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 15 Apr 2019 12:35:05 +0800 Subject: [PATCH 02/34] add log util Former-commit-id: e512880472926020b2d2bf13d8b69347767d872c --- cpp/src/CMakeLists.txt | 1 + cpp/src/main.cpp | 8 ++++++- cpp/src/utils/CommonUtil.cpp | 9 ++++---- cpp/src/utils/Log.h | 41 ++++++++++++++++++++++++++++++++++++ cpp/src/utils/LogUtil.cpp | 24 +++++++++++++++++++++ cpp/src/utils/LogUtil.h | 31 +++++++++++++++++++++++++++ cpp/src/utils/SignalUtil.cpp | 1 + 7 files changed, 110 insertions(+), 5 deletions(-) create mode 100644 cpp/src/utils/Log.h create mode 100644 cpp/src/utils/LogUtil.cpp create mode 100644 cpp/src/utils/LogUtil.h diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 3ab957c9dd..bdaa6c8848 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -20,6 +20,7 @@ add_executable(vecwise_engine_server ${config_files} ${server_files} ${utils_files} + ${VECWISE_THIRD_PARTY_BUILD}/include/easylogging++.cc ) set(dependency_libs diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 5e3b13a181..03ffd03e07 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -10,9 +10,13 @@ #include #include #include +#include #include "utils/SignalUtil.h" #include "utils/CommonUtil.h" +#include "utils/LogUtil.h" + +INITIALIZE_EASYLOGGINGPP void print_help(const std::string &app_name); @@ -20,7 +24,9 @@ using namespace zilliz::vecwise; int main(int argc, char *argv[]) { - server::CommonUtil::PrintInfo("Vecwise engine server start"); + zilliz::vecwise::server::InitLog(); + + printf("Vecwise engine server start...\n"); // zilliz::lib::gpu::InitMemoryAllocator(); diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index da5d011e68..8d991756ad 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -4,6 +4,7 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// #include "CommonUtil.h" +#include "utils/Log.h" #include #include @@ -31,13 +32,13 @@ namespace server { namespace fs = boost::filesystem; void CommonUtil::PrintInfo(const std::string& info){ -// SERVER_LOG_INFO << info; - std::cout << info << std::endl; + SERVER_LOG_INFO << info; +// std::cout << info << std::endl; } void CommonUtil::PrintError(const std::string& info){ -// SERVER_LOG_ERROR << info; - std::cout << info << std::endl; + SERVER_LOG_ERROR << info; +// std::cout << info << std::endl; } bool CommonUtil::GetSystemMemInfo(unsigned long &totalMem, unsigned long &freeMem) { diff --git a/cpp/src/utils/Log.h b/cpp/src/utils/Log.h new file mode 100644 index 0000000000..5d1ef0a487 --- /dev/null +++ b/cpp/src/utils/Log.h @@ -0,0 +1,41 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ +#pragma once + +#include "Error.h" +#include + +namespace zilliz { +namespace vecwise { +namespace server { + +#define SERVER_DOMAIN_NAME "[SERVER] " +#define SERVER_ERROR_TEXT "SERVER Error:" + +#define SERVER_LOG_TRACE LOG(TRACE) << SERVER_DOMAIN_NAME +#define SERVER_LOG_DEBUG LOG(DEBUG) << SERVER_DOMAIN_NAME +#define SERVER_LOG_INFO LOG(INFO) << SERVER_DOMAIN_NAME +#define SERVER_LOG_WARNING LOG(WARNING) << SERVER_DOMAIN_NAME +#define SERVER_LOG_ERROR LOG(ERROR) << SERVER_DOMAIN_NAME +#define SERVER_LOG_FATAL LOG(FATAL) << SERVER_DOMAIN_NAME + +#define SERVER_ERROR(error) \ + ({ \ + SERVER_LOG_ERROR << SERVER_ERROR_TEXT << error; \ + (error); \ + }) + +#define SERVER_CHECK(func) \ + { \ + zilliz::vecwise::server::ServerError error = func; \ + if (error != zilliz::vecwise::server::SERVER_SUCCESS) { \ + return SERVER_ERROR(error); \ + } \ + } \ + +} // namespace sql +} // namespace zilliz +} // namespace server diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp new file mode 100644 index 0000000000..6614667cd4 --- /dev/null +++ b/cpp/src/utils/LogUtil.cpp @@ -0,0 +1,24 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include "LogUtil.h" + +#include + +namespace zilliz { +namespace vecwise { +namespace server { + +int32_t InitLog() { + el::Configurations conf("../../conf/vecwise_engine_log.conf"); + el::Loggers::reconfigureAllLoggers(conf); + + return 0; +} + + +} // server +} // vecwise +} // zilliz diff --git a/cpp/src/utils/LogUtil.h b/cpp/src/utils/LogUtil.h new file mode 100644 index 0000000000..189cbb7370 --- /dev/null +++ b/cpp/src/utils/LogUtil.h @@ -0,0 +1,31 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ +#pragma once + +#include +#include + +namespace zilliz { +namespace vecwise { +namespace server { + +int32_t InitLog(); + +inline std::string GetFileName(std::string filename) { + int pos = filename.find_last_of('/'); + return filename.substr(pos + 1); +} + +#define SHOW_LOCATION +#ifdef SHOW_LOCATION +#define LOCATION_INFO "[" << zilliz::sql::server::GetFileName(__FILE__) << ":" << __LINE__ << "] " +#else +#define LOCATION_INFO "" +#endif + +} +} +} diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index ef83b6c115..8ffa89d942 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -6,6 +6,7 @@ #include "SignalUtil.h" #include "CommonUtil.h" #include "server/Server.h" +#include "utils/Log.h" #include #include From e50f12b59664da131b5e285feee39ff6af1af349 Mon Sep 17 00:00:00 2001 From: jinhai Date: Mon, 15 Apr 2019 15:20:15 +0800 Subject: [PATCH 03/34] Add SQLite ORM Former-commit-id: a496b2e22e605f7c40d6a85eae476f418675cebc From 9fd3827cedfffc5f88c75d2eba3fd8dcfabaa1d5 Mon Sep 17 00:00:00 2001 From: jinhai Date: Mon, 15 Apr 2019 15:21:20 +0800 Subject: [PATCH 04/34] Update README.md and conf Former-commit-id: 9f5a07d7aae8d243fbf6619f0cedfe7a52850237 --- cpp/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/README.md b/cpp/README.md index 1061101dbf..66464cc9cb 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -2,7 +2,7 @@ #### Step 1: install necessery tools centos7 : yum install gfortran - ubunut : sudo apt-install install gfortran + ubuntu16.04 : sudo apt-install install gfortran libsqlite3-dev #### Step 2: build third-parties From 22a100619f1e9747c25f4f765f8df5aa9186e305 Mon Sep 17 00:00:00 2001 From: jinhai Date: Mon, 15 Apr 2019 15:21:56 +0800 Subject: [PATCH 05/34] Update CMakeLists.txt Former-commit-id: fff6e6db7ec57b6ac3d4fd638931e15ee15bf0b7 --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index ff4f8a870e..ca26204603 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -53,7 +53,7 @@ include_directories(${VECWISE_ENGINE_SRC}) include_directories(${VECWISE_THIRD_PARTY_BUILD}/include) link_directories(${CMAKE_CURRRENT_BINARY_DIR}) -link_directories(./third_party/build/lib) +link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib) #execute_process(COMMAND bash build.sh # WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third_party) From b06cf8e121dee8e72ea989085596e76fbb78e4db Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 15 Apr 2019 15:24:56 +0800 Subject: [PATCH 06/34] refine log code Former-commit-id: 42a3cc6e18e85d5e3b8c581ffbbeb4e686ad8be8 --- cpp/conf/server_config.yaml | 30 +++++++++++++++++++++++- cpp/src/main.cpp | 3 --- cpp/src/server/Server.cpp | 16 +++++++------ cpp/src/server/Server.h | 2 -- cpp/src/server/ServerConfig.cpp | 19 ++++++++------- cpp/src/server/ServerConfig.h | 10 ++++---- cpp/src/utils/LogUtil.cpp | 41 +++++++++++++++++++++++++++++++-- 7 files changed, 92 insertions(+), 29 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 6b23da0ec6..d2b78b583c 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -1,3 +1,31 @@ server_config: address: 127.0.0.1 - port: 21001 \ No newline at end of file + port: 21001 + +log_config: + global: + format: "%datetime | %level | %logger | %msg" + filename: "/tmp/logs/vecwise/vecwise_engine-%datetime{%h:%m}-global.log" + enabled: true + to_file: true + to_standard_output: true + subsecond_precision: 3 + performance_tracking: false + max_log_file_size: 2097152 # throw log files away after 2mb + debug: + filename: "/tmp/logs/vecwise/vecwise_engine-%datetime{%h:%m}-debug.log" + enabled: true + warning: + filename: "/tmp/logs/vecwise/vecwise_engine-%datetime{%h:%m}-warning.log" + trace: + filename: "/tmp/logs/vecwise/vecwise_engine-%datetime{%h:%m}-trace.log" + verbose: + format: "%datetime{%d/%m/%y} | %level-%vlevel | %msg" + to_file: false + to_standard_output: true + error: + enabled: false + filename: "/tmp/logs/vecwise/vecwise_engine-%datetime{%h:%m}-error.log" + fatal: + enabled: false + filename: "/tmp/logs/vecwise/vecwise_engine-%datetime{%h:%m}-fatal.log" \ No newline at end of file diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 03ffd03e07..689f7a7af7 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -14,7 +14,6 @@ #include "utils/SignalUtil.h" #include "utils/CommonUtil.h" -#include "utils/LogUtil.h" INITIALIZE_EASYLOGGINGPP @@ -24,8 +23,6 @@ using namespace zilliz::vecwise; int main(int argc, char *argv[]) { - zilliz::vecwise::server::InitLog(); - printf("Vecwise engine server start...\n"); // zilliz::lib::gpu::InitMemoryAllocator(); diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 6d5aa7cafe..0a0f95fb1a 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -4,9 +4,11 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// #include "Server.h" +#include "ServerConfig.h" #include "utils/CommonUtil.h" #include "utils/SignalUtil.h" #include "utils/TimeRecorder.h" +#include "utils/LogUtil.h" #include #include @@ -26,8 +28,7 @@ Server::Instance() { return &server; } -Server::Server() - : opt_config_ptr_(nullptr){ +Server::Server() { } Server::~Server() { @@ -142,12 +143,14 @@ Server::Start() { return 1; } + zilliz::vecwise::server::InitLog(); + //log path is defined by LoadConfig, so InitLog must be called after LoadConfig - ServerConfig *config = ServerConfig::GetInstance(); - ConfigNode server_config = config->GetServerConfig(); + ServerConfig &config = ServerConfig::GetInstance(); + ConfigNode server_config = config.GetConfig(CONFIG_SERVER); //print config into console and log - opt_config_ptr_->PrintAll(); + config.PrintAll(); // Handle Signal signal(SIGINT, SignalUtil::HandleSignal); @@ -206,8 +209,7 @@ Server::Stop() { ServerError Server::LoadConfig() { - opt_config_ptr_ = ServerConfig::GetInstance(); - opt_config_ptr_->LoadConfigFile(config_filename_); + ServerConfig::GetInstance().LoadConfigFile(config_filename_); return SERVER_SUCCESS; } diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index b6ebae1e5b..85478530d1 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -5,7 +5,6 @@ ******************************************************************************/ #pragma once -#include "ServerConfig.h" #include "utils/Error.h" #include @@ -41,7 +40,6 @@ class Server { int pid_fd = -1; std::string pid_filename_; std::string config_filename_; - ServerConfig* opt_config_ptr_ = nullptr; }; // Server } // server diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index 09f4c752dd..ffcfd47181 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -17,10 +17,13 @@ namespace zilliz { namespace vecwise { namespace server { -ServerConfig* +static const std::string CONFIG_ADDRESS = "address"; +static const std::string CONFIG_PORT = "port"; + +ServerConfig& ServerConfig::GetInstance() { static ServerConfig config; - return &config; + return config; } ServerError @@ -63,28 +66,28 @@ ServerConfig::PrintAll() const { } ConfigNode -ServerConfig::GetServerConfig() const { +ServerConfig::GetConfig(const std::string& name) const { const IConfigMgr* mgr = IConfigMgr::GetInstance(); const ConfigNode& root_node = mgr->GetRootNode(); - return root_node.GetChild(CONFIG_SERVER); + return root_node.GetChild(name); } ConfigNode& -ServerConfig::GetServerConfig() { +ServerConfig::GetConfig(const std::string& name) { IConfigMgr* mgr = IConfigMgr::GetInstance(); ConfigNode& root_node = mgr->GetRootNode(); - return root_node.GetChild(CONFIG_SERVER); + return root_node.GetChild(name); } std::string ServerConfig::GetServerAddress() const { - ConfigNode server_config = GetServerConfig(); + ConfigNode server_config = GetConfig(CONFIG_SERVER); return server_config.GetValue(CONFIG_ADDRESS); } std::string ServerConfig::GetServerPort() const { - ConfigNode server_config = GetServerConfig(); + ConfigNode server_config = GetConfig(CONFIG_SERVER); return server_config.GetValue(CONFIG_PORT); } diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index 4277d66066..d3169686bd 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -15,19 +15,17 @@ namespace vecwise { namespace server { static const std::string CONFIG_SERVER = "server_config"; -static const std::string CONFIG_ADDRESS = "address"; -static const std::string CONFIG_PORT = "port"; - +static const std::string CONFIG_LOG = "log_config"; class ServerConfig { public: - static ServerConfig *GetInstance(); + static ServerConfig &GetInstance(); ServerError LoadConfigFile(const std::string& config_filename); void PrintAll() const; - ConfigNode GetServerConfig() const; - ConfigNode& GetServerConfig(); + ConfigNode GetConfig(const std::string& name) const; + ConfigNode& GetConfig(const std::string& name); std::string GetServerAddress() const; std::string GetServerPort() const; diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index 6614667cd4..e7fd0c8596 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -4,17 +4,54 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// #include "LogUtil.h" +#include "server/ServerConfig.h" #include +#include namespace zilliz { namespace vecwise { namespace server { int32_t InitLog() { - el::Configurations conf("../../conf/vecwise_engine_log.conf"); - el::Loggers::reconfigureAllLoggers(conf); +#if 0 + ServerConfig &config = ServerConfig::GetInstance(); + ConfigNode log_config = config.GetConfig(CONFIG_LOG); + const std::map& settings = log_config.GetChildren(); + std::string str_config; + for(auto iter : settings) { + str_config += "* "; + str_config += iter.first; + str_config += ":"; + str_config.append("\n"); + + auto sub_configs = iter.second.GetConfig(); + for(auto it_sub : sub_configs) { + str_config += " "; + str_config += it_sub.first; + str_config += " = "; + std::string temp = it_sub.first; + std::transform(temp.begin(), temp.end(), temp.begin(), ::tolower); + bool is_text = (temp == "format" || temp == "filename"); + if(is_text){ + str_config += "\""; + } + str_config += it_sub.second; + if(is_text){ + str_config += "\""; + } + str_config.append("\n"); + } + } + + el::Configurations conf; + conf.parseFromText(str_config); +#else + el::Configurations conf("../../conf/vecwise_engine_log.conf"); +#endif + + el::Loggers::reconfigureAllLoggers(conf); return 0; } From 018b32bca2762af5f2d48d97491a5ad56a45ff6b Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 15 Apr 2019 15:58:28 +0800 Subject: [PATCH 07/34] add block queue class Former-commit-id: b5fabddb317de5e99462124757e8c1e4466bb184 --- cpp/conf/server_config.yaml | 9 ++- cpp/src/utils/BlockingQueue.h | 54 ++++++++++++++++++ cpp/src/utils/BlockingQueue.inl | 99 +++++++++++++++++++++++++++++++++ cpp/src/utils/Error.h | 23 ++++++++ 4 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 cpp/src/utils/BlockingQueue.h create mode 100644 cpp/src/utils/BlockingQueue.inl diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index d2b78b583c..9b56ac7e85 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -1,6 +1,8 @@ server_config: address: 127.0.0.1 - port: 21001 + port: 33001 + transfer_protocol: json #optional: binary, compact, json, simple_json, debug + server_mode: thread_pool #optional: simple, non_blocking, hsha, thread_pool, thread_selector log_config: global: @@ -28,4 +30,7 @@ log_config: filename: "/tmp/logs/vecwise/vecwise_engine-%datetime{%h:%m}-error.log" fatal: enabled: false - filename: "/tmp/logs/vecwise/vecwise_engine-%datetime{%h:%m}-fatal.log" \ No newline at end of file + filename: "/tmp/logs/vecwise/vecwise_engine-%datetime{%h:%m}-fatal.log" + +cache_config: + cache_capacity: 16 # unit: GB \ No newline at end of file diff --git a/cpp/src/utils/BlockingQueue.h b/cpp/src/utils/BlockingQueue.h new file mode 100644 index 0000000000..cd94997ffd --- /dev/null +++ b/cpp/src/utils/BlockingQueue.h @@ -0,0 +1,54 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ +#pragma once + +#include +#include +#include +#include +#include + +namespace zilliz { +namespace vecwise { +namespace server { + +template +class BlockingQueue { +public: + BlockingQueue() : mtx(), full_(), empty_() {} + + BlockingQueue(const BlockingQueue &rhs) = delete; + + BlockingQueue &operator=(const BlockingQueue &rhs) = delete; + + void Put(const T &task); + + T Take(); + + T Front(); + + T Back(); + + size_t Size(); + + bool Empty(); + + void SetCapacity(const size_t capacity); + +private: + mutable std::mutex mtx; + std::condition_variable full_; + std::condition_variable empty_; + std::queue queue_; + size_t capacity_ = 32; +}; + +} +} +} + + +#include "./BlockingQueue.inl" diff --git a/cpp/src/utils/BlockingQueue.inl b/cpp/src/utils/BlockingQueue.inl new file mode 100644 index 0000000000..f7677ae6c3 --- /dev/null +++ b/cpp/src/utils/BlockingQueue.inl @@ -0,0 +1,99 @@ +#pragma once + +#include "Log.h" +#include "Error.h" + +namespace zilliz { +namespace vecwise { +namespace server { + +template +void +BlockingQueue::Put(const T &task) { + std::unique_lock lock(mtx); + full_.wait(lock, [this] { return (queue_.size() < capacity_); }); + + if (queue_.size() >= capacity_) { + std::string error_msg = + "blocking queue is full, capacity: " + std::to_string(capacity_) + " queue_size: " + + std::to_string(queue_.size()); + SERVER_LOG_ERROR << error_msg; + throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); + } + + queue_.push(task); + empty_.notify_all(); +} + +template +T +BlockingQueue::Take() { + std::unique_lock lock(mtx); + empty_.wait(lock, [this] { return !queue_.empty(); }); + + if (queue_.empty()) { + std::string error_msg = "blocking queue empty"; + SERVER_LOG_ERROR << error_msg; + throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); + } + + T front(queue_.front()); + queue_.pop(); + full_.notify_all(); + return front; +} + +template +size_t +BlockingQueue::Size() { + std::lock_guard lock(mtx); + return queue_.size(); +} + +template +T +BlockingQueue::Front() { + std::unique_lock lock(mtx); + empty_.wait(lock, [this] { return !queue_.empty(); }); + if (queue_.empty()) { + std::string error_msg = "blocking queue empty"; + SERVER_LOG_ERROR << error_msg; + throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); + } + T front(queue_.front()); + return front; +} + +template +T +BlockingQueue::Back() { + std::unique_lock lock(mtx); + empty_.wait(lock, [this] { return !queue_.empty(); }); + + if (queue_.empty()) { + std::string error_msg = "blocking queue empty"; + SERVER_LOG_ERROR << error_msg; + throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); + } + + T back(queue_.back()); + return back; +} + +template +bool +BlockingQueue::Empty() { + std::unique_lock lock(mtx); + return queue_.empty(); +} + +template +void +BlockingQueue::SetCapacity(const size_t capacity) { + capacity_ = (capacity > 0 ? capacity : capacity_); +} + +} +} +} + diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index 6f3de0adaa..83da0cea9f 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -6,6 +6,8 @@ #pragma once #include +#include +#include namespace zilliz { namespace vecwise { @@ -28,6 +30,27 @@ constexpr ServerError SERVER_NULL_POINTER = ToGlobalServerErrorCode(0x003); constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(0x004); constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(0x005); constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(0x006); +constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(0x007); + +class ServerException : public std::exception { +public: + ServerException(ServerError error_code, + const std::string &message = std::string()) + : error_code_(error_code), message_(message) {} + +public: + ServerError error_code() const { + return error_code_; + } + + virtual const char *what() const noexcept { + return message_.c_str(); + } + +private: + ServerError error_code_; + std::string message_; +}; } // namespace server } // namespace vecwise From e7520ebd746ba52c9af87e29230f363cc54d725d Mon Sep 17 00:00:00 2001 From: jinhai Date: Mon, 15 Apr 2019 16:46:26 +0800 Subject: [PATCH 08/34] Remove thrift intermediate files Former-commit-id: 5b063aab4d93c2028e33e860fff02aaa89c35799 From ea0bde2262e13bda2f310e370939e7075a6a0abf Mon Sep 17 00:00:00 2001 From: jinhai Date: Mon, 15 Apr 2019 16:50:58 +0800 Subject: [PATCH 09/34] .gitignore updated Former-commit-id: dc8e455cc15984f3c53e6a75b7ec1df8c2c438f2 --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 56c25e539b..d177a4841d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ __pycache__ build cmake-build-debug cmake_build +*.o +*.lo +*.tar.gz From d7f5e351a2ded17a70ff44a8366cdc585e70fe98 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Sun, 14 Apr 2019 12:56:11 +0800 Subject: [PATCH 10/34] feat(storage): add id generators and memvectors Former-commit-id: 78f7406456556882247c1ac4ccc9bb928699f896 --- cpp/src/storage/id_generators.cpp | 32 ++++++++++++ cpp/src/storage/id_generators.h | 35 +++++++++++++ cpp/src/storage/memvectors.cpp | 84 +++++++++++++++++++++++++++++++ cpp/src/storage/memvectors.h | 56 +++++++++++++++++++++ cpp/src/storage/serializers.h | 11 ++++ 5 files changed, 218 insertions(+) create mode 100644 cpp/src/storage/id_generators.cpp create mode 100644 cpp/src/storage/id_generators.h create mode 100644 cpp/src/storage/memvectors.cpp create mode 100644 cpp/src/storage/memvectors.h create mode 100644 cpp/src/storage/serializers.h diff --git a/cpp/src/storage/id_generators.cpp b/cpp/src/storage/id_generators.cpp new file mode 100644 index 0000000000..8296a4048f --- /dev/null +++ b/cpp/src/storage/id_generators.cpp @@ -0,0 +1,32 @@ +#include +#include + +#inlcude "id_generators.h" + +using std::chrono; + +namespace vecengine { + +IDGenerator::~IDGenerator() {} + +IDNumber SimpleIDGenerator::getNextIDNumber() { + auto now = chrono::system_clock::now(); + auto micros = duration_cast(now.time_since_epoch()).count(); + return micros * MAX_IDS_PER_MICRO +} + +IDNumbers&& SimpleIDGenerator::getNextIDNumbers(size_t n) { + assert(n < MAX_IDS_PER_MICRO); + auto now = chrono::system_clock::now(); + auto micros = duration_cast(now.time_since_epoch()).count(); + micros *= MAX_IDS_PER_MICRO; + + IDNumbers ids = IDNumbers(n); + for (int pos=0; pos + +namespace vecengine { + +#define uint64_t IDNumber; +#define IDNumber* IDNumberPtr; +#define std::vector IDNumbers; + +class IDGenerator { +public: + virtual IDNumber getNextIDNumber() = 0; + virtual IDNumbers&& getNextIDNumbers(size_t n_) = 0; + + virtual ~IDGenerator(); + +}; // IDGenerator + + +class SimpleIDGenerator : public IDGenerator { +public: + virtual IDNumber getNextIDNumber() override; + virtual IDNumbers&& getNextIDNumbers(size_t n_) override; + +private: + const MAX_IDS_PER_MICRO = 1000; + +}; // SimpleIDGenerator + + +} // namespace vecengine + +#endif // UTILS_ID_GENERATORS_H_ diff --git a/cpp/src/storage/memvectors.cpp b/cpp/src/storage/memvectors.cpp new file mode 100644 index 0000000000..3a30dfda9e --- /dev/null +++ b/cpp/src/storage/memvectors.cpp @@ -0,0 +1,84 @@ +#include +#include +#include + +#include "memvectors.h" + + +namespace vecengine { + +MemVectors::MemVectors(size_t dimension_, const std::string& file_location_) : + _file_location(file_location_), + _pIdGenerator(new SimpleIDGenerator()), + _dimension(dimension_), + _pInnerIndex(new faiss::IndexFlat(_dimension)), + _pIdMapIndex = new faiss::IndexIDMap(_pInnerIndex) { +} + +IDNumbers&& MemVectors::add(size_t n, const float* vectors) { + IDNumbers&& ids = _pIdGenerator->getNextIDNumbers(n); + _pIdMapIndex->add_with_ids(n, vectors, pIds, &ids[0]); + return ids; +} + +size_t MemVectors::total() const { + return _pIdMapIndex->ntotal; +} + +size_t MemVectors::approximate_size() const { + return total() * _dimension; +} + +void MemVectors::serialize() { + faiss::write_index(_pIdMapIndex, _file_location); +} + +MemVectors::~MemVectors() { + if (_pIdGenerator != nullptr) { + delete _pIdGenerator; + _pIdGenerator = nullptr; + } + if (_pIdMapIndex != nullptr) { + delete _pIdMapIndex; + _pIdMapIndex = nullptr; + } + if (_pInnerIndex != nullptr) { + delete _pInnerIndex; + _pInnerIndex = nullptr; + } +} + +/* + * MemManager + */ + +MemVectors* MemManager::get_mem_by_group(const std::string& group_id_) { + auto memIt = _memMap.find(group_id_); + if memIt != _memMap.end() { + return &(memIt->second); + } + // PXU TODO: + // 1. Read Group meta info + // 2. Initalize MemVectors base meta info + return nullptr; + /* GroupMetaInfo info; */ + /* bool succ = env->getGroupMeta(group_id, &info); */ + /* if (!succ) { */ + /* return nullptr; */ + /* } */ + /* _memMap[group_id] = MemVectors(info.dimension, info.next_file_location); */ + /* return &(_memMap[group_id]); */ +} + +IDNumbers&& MemManager::add_vectors_no_lock(const std::string& group_id_, + size_t n, + const float* vectors) { + auto mem = get_group_mem(group_id_); + if (mem == nullptr) { + return IDNumbers(); + } + return mem->add(n, vectors); +} + + +} // namespace vecengine diff --git a/cpp/src/storage/memvectors.h b/cpp/src/storage/memvectors.h new file mode 100644 index 0000000000..1a025f8d26 --- /dev/null +++ b/cpp/src/storage/memvectors.h @@ -0,0 +1,56 @@ +#ifndef STORAGE_VECENGINE_MEMVECTORS_H_ +#define STORAGE_VECENGINE_MEMVECTORS_H_ + +#include +#include +#include "id_generators.h" + +class faiss::IndexIDMap; +class faiss::Index; + + +namespace vecengine { + +class MemVectors { +public: + explicit MemVectors(size_t dimension_, const std::string& file_location_); + + IDNumbers&& add(size_t n, const float* vectors); + + size_t total() const; + + size_t approximate_size() const; + + void serialize(); + + ~MemVectors(); + +private: + std::string _file_location; + IDGenerator* _pIdGenerator; + size_t _dimension; + faiss::Index* _pInnerIndex; + faiss::IndexIDMap* _pIdMapIndex; + +}; // MemVectors + + +class MemManager { +public: + MemManager() = default; + + MemVectors* get_mem_by_group(const std::string& group_id_); + +private: + IDNumbers&& add_vectors_no_lock(const std::string& group_id_, + size_t n, + const float* vectors); + + typedef std::map MemMap; + MemMap _memMap; +}; // MemManager + + +} // namespace vecengine + +#endif diff --git a/cpp/src/storage/serializers.h b/cpp/src/storage/serializers.h new file mode 100644 index 0000000000..5356691530 --- /dev/null +++ b/cpp/src/storage/serializers.h @@ -0,0 +1,11 @@ +#ifndef STORAGE_SERIALIZERS_H_ +#define STORAGE_SERIALIZERS_H_ + +class Serializer { +public: + + +}; // Serializer + + +#endif // STORAGE_SERIALIZERS_H_ From 02bcac92a11b78f30026d20bd6c9345f328f4055 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Sun, 14 Apr 2019 14:30:13 +0800 Subject: [PATCH 11/34] feat(cpp): add db and env Former-commit-id: fc2f7219ff7806ca1f98d402ac5120e6174d5e01 --- cpp/src/db.h | 23 +++++++++++++++++++++++ cpp/src/db_impl.cpp | 11 +++++++++++ cpp/src/db_impl.h | 24 ++++++++++++++++++++++++ cpp/src/env.cpp | 11 +++++++++++ cpp/src/env.h | 23 +++++++++++++++++++++++ cpp/src/options.h | 12 ++++++++++++ 6 files changed, 104 insertions(+) create mode 100644 cpp/src/db.h create mode 100644 cpp/src/db_impl.cpp create mode 100644 cpp/src/db_impl.h create mode 100644 cpp/src/env.cpp create mode 100644 cpp/src/env.h create mode 100644 cpp/src/options.h diff --git a/cpp/src/db.h b/cpp/src/db.h new file mode 100644 index 0000000000..5d8c0cf791 --- /dev/null +++ b/cpp/src/db.h @@ -0,0 +1,23 @@ +#ifndef VECENGINE_DB_H_ +#define VECENGINE_DB_H_ + +#include "options.h" + +namespace vecengine { + +class Env; + +class DB { +public: + static DB* Open(const Options& options_, const std::string& name_); + + DB() = default; + DB(const DB&) = delete; + DB& operator=(const DB&) = delete; + + virtual ~DB(); +}; // DB + +} // namespace vecengine + +#endif // VECENGINE_DB_H_ diff --git a/cpp/src/db_impl.cpp b/cpp/src/db_impl.cpp new file mode 100644 index 0000000000..66ee9633ee --- /dev/null +++ b/cpp/src/db_impl.cpp @@ -0,0 +1,11 @@ +include "db_impl.h" + +namespace vecengine { + +DB::DB(const Options& options_, const std::string& name_) + : _dbname(name_), + _env(options_.env), + _options(options_) { +} + +} // namespace vecengine diff --git a/cpp/src/db_impl.h b/cpp/src/db_impl.h new file mode 100644 index 0000000000..7dc55273f1 --- /dev/null +++ b/cpp/src/db_impl.h @@ -0,0 +1,24 @@ +#ifndef VECENGINE_DB_IMPL_H_ +#define VECENGINE_DB_IMPL_H_ + +#include "db.h" + +namespace vecengine { + +class Env; + +class DBImpl : public DB { +public: + DBImpl(const Options& options_, const std::string& name_); + + virtual ~DBImpl(); +private: + const _dbname; + Env* const _env; + const Options _options; + +}; // DBImpl + +} // namespace vecengine + +#endif // VECENGINE_DB_IMPL_H_ diff --git a/cpp/src/env.cpp b/cpp/src/env.cpp new file mode 100644 index 0000000000..fd8088cabf --- /dev/null +++ b/cpp/src/env.cpp @@ -0,0 +1,11 @@ +#inlcude "env.h" + +namespace vecengine { + +DBConfig::DBConfig() + : _mem_sync_interval(10), + _file_merge_trigger_number(20), + _index_file_build_trigger_size(100000) { +} + +} // namespace vecengine diff --git a/cpp/src/env.h b/cpp/src/env.h new file mode 100644 index 0000000000..cda7a55397 --- /dev/null +++ b/cpp/src/env.h @@ -0,0 +1,23 @@ +#ifndef STORAGE_VECENGINE_ENV_H_ +#define STORAGE_VECENGINE_ENV_H_ + +namespace vecengine { + +/* struct Options { */ +/* std::string _db_location; */ +/* size_t _mem_sync_interval; */ +/* size_t _file_merge_trigger_number; */ +/* size_t _index_file_build_trigger_size; */ +/* }; // Config */ + +class Env { +public: + Env() = default; + +private: + Options _option; +}; // Env + +} //namespace vecengine + +#endif // STORAGE_VECENGINE_ENV_H_ diff --git a/cpp/src/options.h b/cpp/src/options.h new file mode 100644 index 0000000000..114861104d --- /dev/null +++ b/cpp/src/options.h @@ -0,0 +1,12 @@ +#ifndef VECENGINE_OPTIONS_H_ +#define VECENGINE_OPTIONS_H_ + +namespace vecengine { + +struct Options { + +}; // Options + +} // namespace vecengine + +#endif // VECENGINE_OPTIONS_H_ From c221786d220463ab523d1350893f186c2da8012e Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Sun, 14 Apr 2019 17:17:33 +0800 Subject: [PATCH 12/34] feat(cpp/db): add meta and status Former-commit-id: 3001a50219a1365ffd0d43fc89cd482344b99ad2 --- cpp/src/db.h | 4 +++ cpp/src/db_impl.cpp | 24 +++++++++++++-- cpp/src/db_impl.h | 8 +++++ cpp/src/db_meta.cpp | 14 +++++++++ cpp/src/db_meta.h | 60 ++++++++++++++++++++++++++++++++++++ cpp/src/db_meta_impl.cpp | 66 ++++++++++++++++++++++++++++++++++++++++ cpp/src/db_meta_impl.h | 45 +++++++++++++++++++++++++++ cpp/src/options.h | 18 +++++++++++ cpp/src/status.cpp | 52 +++++++++++++++++++++++++++++++ cpp/src/status.h | 55 +++++++++++++++++++++++++++++++++ 10 files changed, 344 insertions(+), 2 deletions(-) create mode 100644 cpp/src/db_meta.cpp create mode 100644 cpp/src/db_meta.h create mode 100644 cpp/src/db_meta_impl.cpp create mode 100644 cpp/src/db_meta_impl.h create mode 100644 cpp/src/status.cpp create mode 100644 cpp/src/status.h diff --git a/cpp/src/db.h b/cpp/src/db.h index 5d8c0cf791..1840450502 100644 --- a/cpp/src/db.h +++ b/cpp/src/db.h @@ -1,6 +1,7 @@ #ifndef VECENGINE_DB_H_ #define VECENGINE_DB_H_ +#include #include "options.h" namespace vecengine { @@ -11,6 +12,9 @@ class DB { public: static DB* Open(const Options& options_, const std::string& name_); + virtual std::string add_group(GroupOptions options_, + const std::string& group_id_) = 0; + DB() = default; DB(const DB&) = delete; DB& operator=(const DB&) = delete; diff --git a/cpp/src/db_impl.cpp b/cpp/src/db_impl.cpp index 66ee9633ee..3d030d9e2a 100644 --- a/cpp/src/db_impl.cpp +++ b/cpp/src/db_impl.cpp @@ -1,11 +1,31 @@ -include "db_impl.h" +#include +#include "db_impl.h" namespace vecengine { -DB::DB(const Options& options_, const std::string& name_) +DBImpl::DBImpl(const Options& options_, const std::string& name_) : _dbname(name_), _env(options_.env), _options(options_) { } +Status DBImpl::add_group(const GroupOptions& options_, + const std::string& group_id_, + std::string& gid_) { + assert((!options_.has_id) || + (options_.has_id && ("" != group_id_))); + +} + +/* + * DB + */ + +DB::~DB() {} + +DB* DB::Open(const Options& options_, const std::string& name_) { + DBImpl* impl = new DBImpl(options_, name_); + return impl; +} + } // namespace vecengine diff --git a/cpp/src/db_impl.h b/cpp/src/db_impl.h index 7dc55273f1..0d7d2a3f10 100644 --- a/cpp/src/db_impl.h +++ b/cpp/src/db_impl.h @@ -11,8 +11,16 @@ class DBImpl : public DB { public: DBImpl(const Options& options_, const std::string& name_); + virtual Status add_group(GroupOptions options_, + const std::string& group_id_, + std::string& gid_) override; + virtual ~DBImpl(); private: + + Status meta_add_group(const std::string& group_id_); + Status meta_add_group_file(const std::string& group_id_); + const _dbname; Env* const _env; const Options _options; diff --git a/cpp/src/db_meta.cpp b/cpp/src/db_meta.cpp new file mode 100644 index 0000000000..c1fc0f75f2 --- /dev/null +++ b/cpp/src/db_meta.cpp @@ -0,0 +1,14 @@ +#include "db_meta.h" + +namespace zilliz { +namespace vecwise { +namespace engine { + +Meta* Meta::Default() { + static DefaultMeta meta; + return *meta; +} + +} // namespace engine +} // namespace vecwise +} // namespace zilliz diff --git a/cpp/src/db_meta.h b/cpp/src/db_meta.h new file mode 100644 index 0000000000..e84d97051f --- /dev/null +++ b/cpp/src/db_meta.h @@ -0,0 +1,60 @@ +#ifndef VECENGINE_DB_META_H_ +#define VECENGINE_DB_META_H_ + +namespace zilliz { +namespace vecwise { +namespace engine { + +struct GroupSchema { + size_t id; + std::string group_id; + size_t files_cnt = 0; + uint16_t dimension; + std::string location = ""; +}; // GroupSchema + + +struct GroupFileSchema { + typedef enum { + RAW, + INDEX + } FILE_TYPE; + + size_t id; + std::string group_id; + std::string file_id; + int files_type = RAW; + size_t rows; + std::string location = ""; +}; // GroupFileSchema + +typedef std::vector GroupFilesSchema; + + +class Meta { +public: + virtual Status add_group(const std::string& group_id_, GroupSchema& group_info) = 0; + virtual Status get_group(const std::string& group_id_, GroupSchema& group_info) = 0; + virtual Status has_group(const std::string& group_id_, bool& has_or_not) = 0; + + virtual Status add_group_file(const std::string& group_id, + GroupFileSchema& group_file_info) = 0; + virtual Status has_group_file(const std::string& group_id, + const std::string& file_id, + bool& has_or_not) = 0; + virtual Status get_group_file(const std::string& group_id, + const std::string& file_id, + GroupFileSchema& group_file_info) = 0; + virtual Status mark_group_file_as_index(const std::string& group_id, + const std::string& file_id) = 0; + + virtual Status get_group_files(const std::string& group_id, + GroupFilesSchema& group_files_info) = 0; + +}; // MetaData + +} // namespace engine +} // namespace vecwise +} // namespace zilliz + +#endif // VECENGINE_DB_META_H_ diff --git a/cpp/src/db_meta_impl.cpp b/cpp/src/db_meta_impl.cpp new file mode 100644 index 0000000000..a6754a817d --- /dev/null +++ b/cpp/src/db_meta_impl.cpp @@ -0,0 +1,66 @@ +#include "db_meta_impl.h" + +namespace zilliz { +namespace vecwise { +namespace engine { + +Status DBMetaImpl::DBMetaImpl(DBMetaOptions options_) + : _options(options_) { + initialize(); +} + +Status DBMetaImpl::initialize() { + // PXU TODO: Create DB Connection + return Status.OK(); +} + +Status DBMetaImpl::add_group(const std::string& group_id_, GroupSchema& group_info) { + //PXU TODO + return Status.OK(); +} + +Status DBMetaImpl::get_group(const std::string& group_id_, GroupSchema& group_info) { + //PXU TODO + return Status.OK(); +} + +Status DBMetaImpl::has_group(const std::string& group_id_, bool& has_or_not) { + //PXU TODO + return Status.OK(); +} + +Status DBMetaImpl::add_group_file(const std::string& group_id, + GroupFileSchema& group_file_info) { + //PXU TODO + return Status.OK(); +} + +Status DBMetaImpl::has_group_file(const std::string& group_id, + const std::string& file_id, + bool& has_or_not) { + //PXU TODO + return Status.OK(); +} + +Status DBMetaImpl::get_group_file(const std::string& group_id, + const std::string& file_id, + GroupFileSchema& group_file_info) { + //PXU TODO + return Status.OK(); +} + +Status DBMetaImpl::get_group_files(const std::string& group_id, + GroupFilesSchema& group_files_info) { + // PXU TODO + return Status.OK(); +} + +Status DBMetaImpl::mark_group_file_as_index(const std::string& group_id, + const std::string& file_id) { + //PXU TODO + return Status.OK(); +} + +} // namespace engine +} // namespace vecwise +} // namespace zilliz diff --git a/cpp/src/db_meta_impl.h b/cpp/src/db_meta_impl.h new file mode 100644 index 0000000000..d3aa5e19d6 --- /dev/null +++ b/cpp/src/db_meta_impl.h @@ -0,0 +1,45 @@ +#ifndef VECENGINE_DB_META_IMPL_H_ +#define VECENGINE_DB_META_IMPL_H_ + +#include "db_meta.h" +#include "options.h" + +namespace zilliz { +namespace vecwise { +namespace engine { + +class DBMetaImpl : public Meta { +public: + DBMetaImpl(DBMetaOptions& options_); + + virtual Status add_group(const std::string& group_id_, GroupSchema& group_info) override; + virtual Status get_group(const std::string& group_id_, GroupSchema& group_info) override; + virtual Status has_group(const std::string& group_id_, bool& has_or_not) override; + + virtual Status add_group_file(const std::string& group_id, + GroupFileSchema& group_file_info) override; + virtual Status has_group_file(const std::string& group_id, + const std::string& file_id, + bool& has_or_not) override; + virtual Status get_group_file(const std::string& group_id, + const std::string& file_id, + GroupFileSchema& group_file_info) override; + virtual Status mark_group_file_as_index(const std::string& group_id, + const std::string& file_id) override; + + virtual Status get_group_files(const std::string& group_id, + GroupFilesSchema& group_files_info) override; + +private: + + Status initialize(); + + const DBMetaOptions _options; + +}; // DBMetaImpl + +} // namespace engine +} // namespace vecwise +} // namespace zilliz + +#endif // VECENGINE_DB_META_IMPL_H_ diff --git a/cpp/src/options.h b/cpp/src/options.h index 114861104d..f07f10cd39 100644 --- a/cpp/src/options.h +++ b/cpp/src/options.h @@ -1,12 +1,30 @@ #ifndef VECENGINE_OPTIONS_H_ #define VECENGINE_OPTIONS_H_ +#include + namespace vecengine { struct Options { }; // Options + +struct GroupOptions { + size_t dimension; + bool has_id = false; +}; // GroupOptions + + +struct MetaOptions { +}; // MetaOptions + + +struct DBMetaOptions : public MetaOptions { + std::string db_uri; +}; // DBMetaOptions + + } // namespace vecengine #endif // VECENGINE_OPTIONS_H_ diff --git a/cpp/src/status.cpp b/cpp/src/status.cpp new file mode 100644 index 0000000000..8c547d5abe --- /dev/null +++ b/cpp/src/status.cpp @@ -0,0 +1,52 @@ +#include "status.h" + +namespace vecengine { + +const char* Status::CopyState(const char* state_) { + uint32_t size; + memcpy(&size, state_, sizeof(size)); + char result = new char[size+5]; + memcpy(result, state_, size+5); + return result; +} + +Status::Status(Code code_, const std::string& msg_, const std::string& msg2_) { + assert(code_ != kOK); + const uint32_t len1 = msg_.size(); + const uint32_t len2 = msg2_.size(); + const uint32_t size = len1 + (len2 ? (2+len2) : 0); + char* result = new char[size+5]; + memcpy(result, &size, sizeof(size)); + result[4] = static_cast(code); + memcpy(result+5, msg_.data(), len1); + if (len2) { + result[5 + len1] = ':'; + result[6 + len1] = ' '; + memcpy(result + 7 + len1, msg2_.data(), len2); + } + _state = result; +} + +std::string Status::ToString() const { + if (_state == nullptr) return "OK"; + char tmp[30]; + const char* type; + switch (code()) { + case kOK: + type = "OK"; + break; + default: + snprintf(tmp, sizeof(tmp), "Unkown code(%d): ", + static_cast(code())); + type = tmp; + break; + } + + std::string result(type); + uint32_t length; + memcpy(&length, state_, sizeof(length)); + result.append(state_ + 5, length); + return result; +} + +} // namespace vecengine diff --git a/cpp/src/status.h b/cpp/src/status.h new file mode 100644 index 0000000000..01e3a4ce5d --- /dev/null +++ b/cpp/src/status.h @@ -0,0 +1,55 @@ +#ifndef VECENGINE_STATUS_H_ +#define VECENGINE_STATUS_H_ + +namespace vecengine { + +class Status { +public: + Status() noexcept : _state(nullptr) {} + ~Status() { delete[] _state; } + + Status(const Status& rhs_); + Status& operator=(const Status& rhs_); + + Status(const Status&& rhs_) noexcept : _state(rhs_._state) { rhs_._state = nullptr; } + Status& operator=(const Status& rhs_) noexcept; + + static Status OK() { return Status(); } + + bool ok() const { return _state == nullptr; } + +private: + const char* _state; + + enum Code { + kOK = 0, + }; + + Code code() const { + return (_state == nullptr) ? kOK : static_cast(_state[4]) + } + + static const char* CopyState(const char* s); + +}; // Status + +inline Status::Status(const Status* rhs_) { + _state = (rhs_._state == nullptr) ? nullptr : CopyState(rhs_._state); +} + +inline Status& Status::operator=(const Status& rhs_) { + if (_state != rhs_._state) { + delete[] state_; + _state = (rhs_._state == nullptr) ? nullptr : CopyState(rhs_._state); + } + return *this; +} + +inline Status& Status::operator=(Status&& rhs_) noexcept { + std::swap(_state, rhs_._state); + return *this; +} + +} // namespace vecengine + +#endif // VECENGINE_STATUS_H_ From adbb6fdaaf800fda44d6dcce20b965751ba00183 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Sun, 14 Apr 2019 17:19:24 +0800 Subject: [PATCH 13/34] refactor(cpp/db): move file locations to db dir Former-commit-id: ff490787d981b2a675bdec5ac5331851b01bf330 --- cpp/src/{ => db}/db.h | 0 cpp/src/{ => db}/db_impl.cpp | 0 cpp/src/{ => db}/db_impl.h | 0 cpp/src/{ => db}/db_meta.cpp | 0 cpp/src/{ => db}/db_meta.h | 0 cpp/src/{ => db}/db_meta_impl.cpp | 0 cpp/src/{ => db}/db_meta_impl.h | 0 cpp/src/{ => db}/env.cpp | 0 cpp/src/{ => db}/env.h | 0 cpp/src/{ => db}/options.h | 0 cpp/src/{ => db}/status.cpp | 0 cpp/src/{ => db}/status.h | 0 12 files changed, 0 insertions(+), 0 deletions(-) rename cpp/src/{ => db}/db.h (100%) rename cpp/src/{ => db}/db_impl.cpp (100%) rename cpp/src/{ => db}/db_impl.h (100%) rename cpp/src/{ => db}/db_meta.cpp (100%) rename cpp/src/{ => db}/db_meta.h (100%) rename cpp/src/{ => db}/db_meta_impl.cpp (100%) rename cpp/src/{ => db}/db_meta_impl.h (100%) rename cpp/src/{ => db}/env.cpp (100%) rename cpp/src/{ => db}/env.h (100%) rename cpp/src/{ => db}/options.h (100%) rename cpp/src/{ => db}/status.cpp (100%) rename cpp/src/{ => db}/status.h (100%) diff --git a/cpp/src/db.h b/cpp/src/db/db.h similarity index 100% rename from cpp/src/db.h rename to cpp/src/db/db.h diff --git a/cpp/src/db_impl.cpp b/cpp/src/db/db_impl.cpp similarity index 100% rename from cpp/src/db_impl.cpp rename to cpp/src/db/db_impl.cpp diff --git a/cpp/src/db_impl.h b/cpp/src/db/db_impl.h similarity index 100% rename from cpp/src/db_impl.h rename to cpp/src/db/db_impl.h diff --git a/cpp/src/db_meta.cpp b/cpp/src/db/db_meta.cpp similarity index 100% rename from cpp/src/db_meta.cpp rename to cpp/src/db/db_meta.cpp diff --git a/cpp/src/db_meta.h b/cpp/src/db/db_meta.h similarity index 100% rename from cpp/src/db_meta.h rename to cpp/src/db/db_meta.h diff --git a/cpp/src/db_meta_impl.cpp b/cpp/src/db/db_meta_impl.cpp similarity index 100% rename from cpp/src/db_meta_impl.cpp rename to cpp/src/db/db_meta_impl.cpp diff --git a/cpp/src/db_meta_impl.h b/cpp/src/db/db_meta_impl.h similarity index 100% rename from cpp/src/db_meta_impl.h rename to cpp/src/db/db_meta_impl.h diff --git a/cpp/src/env.cpp b/cpp/src/db/env.cpp similarity index 100% rename from cpp/src/env.cpp rename to cpp/src/db/env.cpp diff --git a/cpp/src/env.h b/cpp/src/db/env.h similarity index 100% rename from cpp/src/env.h rename to cpp/src/db/env.h diff --git a/cpp/src/options.h b/cpp/src/db/options.h similarity index 100% rename from cpp/src/options.h rename to cpp/src/db/options.h diff --git a/cpp/src/status.cpp b/cpp/src/db/status.cpp similarity index 100% rename from cpp/src/status.cpp rename to cpp/src/db/status.cpp diff --git a/cpp/src/status.h b/cpp/src/db/status.h similarity index 100% rename from cpp/src/status.h rename to cpp/src/db/status.h From de4a44aaa13fad490ab6ad90e6d4bd90fb9c7f89 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Sun, 14 Apr 2019 17:30:19 +0800 Subject: [PATCH 14/34] refactor(cpp/db): arguments name refactor Former-commit-id: ecfcbcc389f2996236521adc77ee6021950ae2d9 --- cpp/src/db/db_meta.cpp | 3 --- cpp/src/db/db_meta.h | 30 +++++++++++++++--------------- cpp/src/db/db_meta_impl.cpp | 30 +++++++++++++++--------------- cpp/src/db/db_meta_impl.h | 30 +++++++++++++++--------------- cpp/src/db/options.h | 3 ++- 5 files changed, 47 insertions(+), 49 deletions(-) diff --git a/cpp/src/db/db_meta.cpp b/cpp/src/db/db_meta.cpp index c1fc0f75f2..42c1d9d78f 100644 --- a/cpp/src/db/db_meta.cpp +++ b/cpp/src/db/db_meta.cpp @@ -4,9 +4,6 @@ namespace zilliz { namespace vecwise { namespace engine { -Meta* Meta::Default() { - static DefaultMeta meta; - return *meta; } } // namespace engine diff --git a/cpp/src/db/db_meta.h b/cpp/src/db/db_meta.h index e84d97051f..7950bbfa16 100644 --- a/cpp/src/db/db_meta.h +++ b/cpp/src/db/db_meta.h @@ -33,23 +33,23 @@ typedef std::vector GroupFilesSchema; class Meta { public: - virtual Status add_group(const std::string& group_id_, GroupSchema& group_info) = 0; - virtual Status get_group(const std::string& group_id_, GroupSchema& group_info) = 0; - virtual Status has_group(const std::string& group_id_, bool& has_or_not) = 0; + virtual Status add_group(const std::string& group_id_, GroupSchema& group_info_) = 0; + virtual Status get_group(const std::string& group_id_, GroupSchema& group_info_) = 0; + virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; - virtual Status add_group_file(const std::string& group_id, - GroupFileSchema& group_file_info) = 0; - virtual Status has_group_file(const std::string& group_id, - const std::string& file_id, - bool& has_or_not) = 0; - virtual Status get_group_file(const std::string& group_id, - const std::string& file_id, - GroupFileSchema& group_file_info) = 0; - virtual Status mark_group_file_as_index(const std::string& group_id, - const std::string& file_id) = 0; + virtual Status add_group_file(const std::string& group_id_, + GroupFileSchema& group_file_info_) = 0; + virtual Status has_group_file(const std::string& group_id_, + const std::string& file_id_, + bool& has_or_not_) = 0; + virtual Status get_group_file(const std::string& group_id_, + const std::string& file_id_, + GroupFileSchema& group_file_info_) = 0; + virtual Status mark_group_file_as_index(const std::string& group_id_, + const std::string& file_id_) = 0; - virtual Status get_group_files(const std::string& group_id, - GroupFilesSchema& group_files_info) = 0; + virtual Status get_group_files(const std::string& group_id_, + GroupFilesSchema& group_files_info_) = 0; }; // MetaData diff --git a/cpp/src/db/db_meta_impl.cpp b/cpp/src/db/db_meta_impl.cpp index a6754a817d..6acbc6aa81 100644 --- a/cpp/src/db/db_meta_impl.cpp +++ b/cpp/src/db/db_meta_impl.cpp @@ -14,49 +14,49 @@ Status DBMetaImpl::initialize() { return Status.OK(); } -Status DBMetaImpl::add_group(const std::string& group_id_, GroupSchema& group_info) { +Status DBMetaImpl::add_group(const std::string& group_id_, GroupSchema& group_info_) { //PXU TODO return Status.OK(); } -Status DBMetaImpl::get_group(const std::string& group_id_, GroupSchema& group_info) { +Status DBMetaImpl::get_group(const std::string& group_id_, GroupSchema& group_info_) { //PXU TODO return Status.OK(); } -Status DBMetaImpl::has_group(const std::string& group_id_, bool& has_or_not) { +Status DBMetaImpl::has_group(const std::string& group_id_, bool& has_or_not_) { //PXU TODO return Status.OK(); } -Status DBMetaImpl::add_group_file(const std::string& group_id, - GroupFileSchema& group_file_info) { +Status DBMetaImpl::add_group_file(const std::string& group_id_, + GroupFileSchema& group_file_info_) { //PXU TODO return Status.OK(); } -Status DBMetaImpl::has_group_file(const std::string& group_id, - const std::string& file_id, - bool& has_or_not) { +Status DBMetaImpl::has_group_file(const std::string& group_id_, + const std::string& file_id_, + bool& has_or_not_) { //PXU TODO return Status.OK(); } -Status DBMetaImpl::get_group_file(const std::string& group_id, - const std::string& file_id, - GroupFileSchema& group_file_info) { +Status DBMetaImpl::get_group_file(const std::string& group_id_, + const std::string& file_id_, + GroupFileSchema& group_file_info_) { //PXU TODO return Status.OK(); } -Status DBMetaImpl::get_group_files(const std::string& group_id, - GroupFilesSchema& group_files_info) { +Status DBMetaImpl::get_group_files(const std::string& group_id_, + GroupFilesSchema& group_files_info_) { // PXU TODO return Status.OK(); } -Status DBMetaImpl::mark_group_file_as_index(const std::string& group_id, - const std::string& file_id) { +Status DBMetaImpl::mark_group_file_as_index(const std::string& group_id_, + const std::string& file_id_) { //PXU TODO return Status.OK(); } diff --git a/cpp/src/db/db_meta_impl.h b/cpp/src/db/db_meta_impl.h index d3aa5e19d6..f373a38ca9 100644 --- a/cpp/src/db/db_meta_impl.h +++ b/cpp/src/db/db_meta_impl.h @@ -12,23 +12,23 @@ class DBMetaImpl : public Meta { public: DBMetaImpl(DBMetaOptions& options_); - virtual Status add_group(const std::string& group_id_, GroupSchema& group_info) override; - virtual Status get_group(const std::string& group_id_, GroupSchema& group_info) override; - virtual Status has_group(const std::string& group_id_, bool& has_or_not) override; + virtual Status add_group(const std::string& group_id_, GroupSchema& group_info_) override; + virtual Status get_group(const std::string& group_id_, GroupSchema& group_info_) override; + virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; - virtual Status add_group_file(const std::string& group_id, - GroupFileSchema& group_file_info) override; - virtual Status has_group_file(const std::string& group_id, - const std::string& file_id, - bool& has_or_not) override; - virtual Status get_group_file(const std::string& group_id, - const std::string& file_id, - GroupFileSchema& group_file_info) override; - virtual Status mark_group_file_as_index(const std::string& group_id, - const std::string& file_id) override; + virtual Status add_group_file(const std::string& group_id_, + GroupFileSchema& group_file_info_) override; + virtual Status has_group_file(const std::string& group_id_, + const std::string& file_id_, + bool& has_or_not_) override; + virtual Status get_group_file(const std::string& group_id_, + const std::string& file_id_, + GroupFileSchema& group_file_info_) override; + virtual Status mark_group_file_as_index(const std::string& group_id_, + const std::string& file_id_) override; - virtual Status get_group_files(const std::string& group_id, - GroupFilesSchema& group_files_info) override; + virtual Status get_group_files(const std::string& group_id_, + GroupFilesSchema& group_files_info_) override; private: diff --git a/cpp/src/db/options.h b/cpp/src/db/options.h index f07f10cd39..9c4b56108b 100644 --- a/cpp/src/db/options.h +++ b/cpp/src/db/options.h @@ -21,7 +21,8 @@ struct MetaOptions { struct DBMetaOptions : public MetaOptions { - std::string db_uri; + std::string backend_uri; + std::string dbname; }; // DBMetaOptions From 2528e61ec3fa73e50a4921ed05fd32fffb72dcc4 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Sun, 14 Apr 2019 18:24:21 +0800 Subject: [PATCH 15/34] feat(db/env): add threading related schedule Former-commit-id: 15409f00183e3727d771354503e8b4665f0c821d --- cpp/src/db/env.cpp | 52 ++++++++++++++++++++++++++++++++++++++------ cpp/src/db/env.h | 48 ++++++++++++++++++++++++++++++---------- cpp/src/db/options.h | 4 +++- 3 files changed, 85 insertions(+), 19 deletions(-) diff --git a/cpp/src/db/env.cpp b/cpp/src/db/env.cpp index fd8088cabf..8b1a288f9f 100644 --- a/cpp/src/db/env.cpp +++ b/cpp/src/db/env.cpp @@ -1,11 +1,49 @@ -#inlcude "env.h" +#include "env.h" -namespace vecengine { +namespace zilliz { +namespace vecwise { +namespace engine { -DBConfig::DBConfig() - : _mem_sync_interval(10), - _file_merge_trigger_number(20), - _index_file_build_trigger_size(100000) { +Env::Env() + : _bg_work_cv(&_bg_work_mutex), + _bg_work_started(false) { } -} // namespace vecengine +void Env::schedule(void (*function_)(void* arg_), void* arg_) { + std::lock_guard lock; + + if (!_bg_work_started) { + _bg_work_started = true; + std::thread bg_thread(Env::BackgroundThreadEntryPoint, this); + bg_thread.detach(); + } + + if (_bg_work_queue.empty()) { + _bg_work_cv.notify_one(); + } + + _bg_work_queue.emplace(function_, arg_); +} + +void Env::backgroud_thread_main() { + while (true) { + std::lock_guard lock; + while (_bg_work_queue.empty()) { + _bg_work_cv.wait(); + } + + assert(!_bg_work_queue.empty()); + auto bg_function = _bg_work_queue.front()._function; + void* bg_arg = _bg_work_queue.front()._arg; + _bg_work_queue.pop(); + + lock.unlock(); + bg_function(bg_arg); + } +} + +Env::~Env() {} + +} // namespace engine +} // namespace vecwise +} // namespace zilliz diff --git a/cpp/src/db/env.h b/cpp/src/db/env.h index cda7a55397..2d2b630606 100644 --- a/cpp/src/db/env.h +++ b/cpp/src/db/env.h @@ -1,23 +1,49 @@ #ifndef STORAGE_VECENGINE_ENV_H_ #define STORAGE_VECENGINE_ENV_H_ -namespace vecengine { +#include +#include +#include +#include -/* struct Options { */ -/* std::string _db_location; */ -/* size_t _mem_sync_interval; */ -/* size_t _file_merge_trigger_number; */ -/* size_t _index_file_build_trigger_size; */ -/* }; // Config */ +namespace zilliz { +namespace vecwise { +namespace engine { class Env { public: - Env() = default; + Env(); + + Env(const Env&) = delete; + Env& operator=(const Env&) = delete; + + void schedule(void (*function_)(void* arg_), void* arg_); + + virtual ~Env(); + +protected: + void backgroud_thread_main(); + static void BackgroundThreadEntryPoint(Env* env) { + env->backgroud_thread_main(); + } + + struct BGWork { + explicit BGWork(void (*function_)(void*), void* arg_) + : _function(function_), _arg(arg_) {} + + void (* const _function)(void*); + void* const _arg; + }; + + std::mutex _bg_work_mutex; + std::condition_variable _bg_work_cv; + std::queue _bg_work_queue; + bool _bg_work_started; -private: - Options _option; }; // Env -} //namespace vecengine +} // namespace engine +} // namespace vecwise +} // namespace zilliz #endif // STORAGE_VECENGINE_ENV_H_ diff --git a/cpp/src/db/options.h b/cpp/src/db/options.h index 9c4b56108b..3e74d0e9b4 100644 --- a/cpp/src/db/options.h +++ b/cpp/src/db/options.h @@ -6,7 +6,9 @@ namespace vecengine { struct Options { - + uint16_t memory_sync_interval = 10; + uint16_t raw_file_merge_trigger_number = 100; + size_t raw_to_index_trigger_size = 100000; }; // Options From 4897341b180c2a2f6219c7b3f43abb2cfdba757f Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Sun, 14 Apr 2019 18:51:08 +0800 Subject: [PATCH 16/34] feat(cpp/db): add background handling Former-commit-id: e2a992049e4f736347a3a549128885f83ed9bfd7 --- cpp/src/db/db_impl.cpp | 33 ++++++++++++++++++++++++++++++++- cpp/src/db/db_impl.h | 13 +++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/cpp/src/db/db_impl.cpp b/cpp/src/db/db_impl.cpp index 3d030d9e2a..3fdbdf01e0 100644 --- a/cpp/src/db/db_impl.cpp +++ b/cpp/src/db/db_impl.cpp @@ -6,7 +6,9 @@ namespace vecengine { DBImpl::DBImpl(const Options& options_, const std::string& name_) : _dbname(name_), _env(options_.env), - _options(options_) { + _options(options_), + _bg_work_finish_signal(_mutex), + _bg_compaction_scheduled(false) { } Status DBImpl::add_group(const GroupOptions& options_, @@ -17,6 +19,35 @@ Status DBImpl::add_group(const GroupOptions& options_, } +void DBImpl::try_schedule_compaction() { + if (_bg_compaction_scheduled) return; + if (!_bg_error.ok()) return; + + _bg_compaction_scheduled = true; + _env->schedule(&DBImpl::BGWork, this); +} + +void DBImpl::BGWork(void* db_) { + reinterpret_cast(db_)->background_call(); +} + +void DBImpl::background_call() { + std::lock_guard lock(_mutex); + assert(_bg_compaction_scheduled); + + if (!_bg_error.ok()) return; + + background_compaction(); +} + +void DBImpl::background_compaction() { + +} + +void DBImpl::compact_memory() { + +} + /* * DB */ diff --git a/cpp/src/db/db_impl.h b/cpp/src/db/db_impl.h index 0d7d2a3f10..de50cda91a 100644 --- a/cpp/src/db/db_impl.h +++ b/cpp/src/db/db_impl.h @@ -1,6 +1,8 @@ #ifndef VECENGINE_DB_IMPL_H_ #define VECENGINE_DB_IMPL_H_ +#include +#include #include "db.h" namespace vecengine { @@ -15,9 +17,15 @@ public: const std::string& group_id_, std::string& gid_) override; + void try_schedule_compaction(); + virtual ~DBImpl(); private: + static void BGWork(void* db); + void background_call(); + void background_compaction(); + Status meta_add_group(const std::string& group_id_); Status meta_add_group_file(const std::string& group_id_); @@ -25,6 +33,11 @@ private: Env* const _env; const Options _options; + std::mutex _mutex; + std::condition_variable _bg_work_finish_signal; + bool _bg_compaction_scheduled; + Status _bg_error; + }; // DBImpl } // namespace vecengine From 20cabdad348639e366d6edd95a0ea2b1fddaea38 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Sun, 14 Apr 2019 18:59:10 +0800 Subject: [PATCH 17/34] refactor(cpp/db): move files from storage to db Former-commit-id: 2da3f22922942552352e5895de1f285aa1de9a70 --- cpp/src/{storage => db}/id_generators.cpp | 0 cpp/src/{storage => db}/id_generators.h | 0 cpp/src/{storage => db}/memvectors.cpp | 0 cpp/src/{storage => db}/memvectors.h | 0 cpp/src/{storage => db}/serializers.h | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename cpp/src/{storage => db}/id_generators.cpp (100%) rename cpp/src/{storage => db}/id_generators.h (100%) rename cpp/src/{storage => db}/memvectors.cpp (100%) rename cpp/src/{storage => db}/memvectors.h (100%) rename cpp/src/{storage => db}/serializers.h (100%) diff --git a/cpp/src/storage/id_generators.cpp b/cpp/src/db/id_generators.cpp similarity index 100% rename from cpp/src/storage/id_generators.cpp rename to cpp/src/db/id_generators.cpp diff --git a/cpp/src/storage/id_generators.h b/cpp/src/db/id_generators.h similarity index 100% rename from cpp/src/storage/id_generators.h rename to cpp/src/db/id_generators.h diff --git a/cpp/src/storage/memvectors.cpp b/cpp/src/db/memvectors.cpp similarity index 100% rename from cpp/src/storage/memvectors.cpp rename to cpp/src/db/memvectors.cpp diff --git a/cpp/src/storage/memvectors.h b/cpp/src/db/memvectors.h similarity index 100% rename from cpp/src/storage/memvectors.h rename to cpp/src/db/memvectors.h diff --git a/cpp/src/storage/serializers.h b/cpp/src/db/serializers.h similarity index 100% rename from cpp/src/storage/serializers.h rename to cpp/src/db/serializers.h From 0d20f3975a8dec92b3d074a17f891bbee81ba193 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Sun, 14 Apr 2019 19:03:39 +0800 Subject: [PATCH 18/34] refactor(cpp/db): unified namespace Former-commit-id: 7f20ca04dbd59f1c33919a410123d1e4af1ec47f --- cpp/src/db/db.h | 8 ++++++-- cpp/src/db/db_impl.h | 10 ++++++++-- cpp/src/db/id_generators.h | 8 ++++++-- cpp/src/db/memvectors.h | 8 ++++++-- cpp/src/db/options.h | 8 ++++++-- cpp/src/db/status.h | 8 ++++++-- 6 files changed, 38 insertions(+), 12 deletions(-) diff --git a/cpp/src/db/db.h b/cpp/src/db/db.h index 1840450502..9707014e0f 100644 --- a/cpp/src/db/db.h +++ b/cpp/src/db/db.h @@ -4,7 +4,9 @@ #include #include "options.h" -namespace vecengine { +namespace zilliz { +namespace vecwise { +namespace engine { class Env; @@ -22,6 +24,8 @@ public: virtual ~DB(); }; // DB -} // namespace vecengine +} // namespace engine +} // namespace vecwise +} // namespace zilliz #endif // VECENGINE_DB_H_ diff --git a/cpp/src/db/db_impl.h b/cpp/src/db/db_impl.h index de50cda91a..a502c0e828 100644 --- a/cpp/src/db/db_impl.h +++ b/cpp/src/db/db_impl.h @@ -5,7 +5,9 @@ #include #include "db.h" -namespace vecengine { +namespace zilliz { +namespace vecwise { +namespace engine { class Env; @@ -40,6 +42,10 @@ private: }; // DBImpl -} // namespace vecengine +} // namespace engine +} // namespace vecwise +} // namespace zilliz + +#endif // VECENGINE_DB_META_IMPL_H_ #endif // VECENGINE_DB_IMPL_H_ diff --git a/cpp/src/db/id_generators.h b/cpp/src/db/id_generators.h index 830b599a2a..767c3ce322 100644 --- a/cpp/src/db/id_generators.h +++ b/cpp/src/db/id_generators.h @@ -3,7 +3,9 @@ #include -namespace vecengine { +namespace zilliz { +namespace vecwise { +namespace engine { #define uint64_t IDNumber; #define IDNumber* IDNumberPtr; @@ -30,6 +32,8 @@ private: }; // SimpleIDGenerator -} // namespace vecengine +} // namespace engine +} // namespace vecwise +} // namespace zilliz #endif // UTILS_ID_GENERATORS_H_ diff --git a/cpp/src/db/memvectors.h b/cpp/src/db/memvectors.h index 1a025f8d26..159ee441d4 100644 --- a/cpp/src/db/memvectors.h +++ b/cpp/src/db/memvectors.h @@ -9,7 +9,9 @@ class faiss::IndexIDMap; class faiss::Index; -namespace vecengine { +namespace zilliz { +namespace vecwise { +namespace engine { class MemVectors { public: @@ -51,6 +53,8 @@ private: }; // MemManager -} // namespace vecengine +} // namespace engine +} // namespace vecwise +} // namespace zilliz #endif diff --git a/cpp/src/db/options.h b/cpp/src/db/options.h index 3e74d0e9b4..283c1905fc 100644 --- a/cpp/src/db/options.h +++ b/cpp/src/db/options.h @@ -3,7 +3,9 @@ #include -namespace vecengine { +namespace zilliz { +namespace vecwise { +namespace engine { struct Options { uint16_t memory_sync_interval = 10; @@ -28,6 +30,8 @@ struct DBMetaOptions : public MetaOptions { }; // DBMetaOptions -} // namespace vecengine +} // namespace engine +} // namespace vecwise +} // namespace zilliz #endif // VECENGINE_OPTIONS_H_ diff --git a/cpp/src/db/status.h b/cpp/src/db/status.h index 01e3a4ce5d..3133c64924 100644 --- a/cpp/src/db/status.h +++ b/cpp/src/db/status.h @@ -1,7 +1,9 @@ #ifndef VECENGINE_STATUS_H_ #define VECENGINE_STATUS_H_ -namespace vecengine { +namespace zilliz { +namespace vecwise { +namespace engine { class Status { public: @@ -50,6 +52,8 @@ inline Status& Status::operator=(Status&& rhs_) noexcept { return *this; } -} // namespace vecengine +} // namespace engine +} // namespace vecwise +} // namespace zilliz #endif // VECENGINE_STATUS_H_ From e06bcc478793da56388c400db710d069ec3adce6 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Sun, 14 Apr 2019 19:28:56 +0800 Subject: [PATCH 19/34] feat(cpp/db): meta schema update Former-commit-id: cb4b6a2b56baf733fe435ce6980477c62cd29331 --- cpp/src/db/db.h | 21 +++++++++++++++++++-- cpp/src/db/db_impl.h | 4 ++++ cpp/src/db/db_meta.h | 7 ++++--- cpp/src/db/db_meta_impl.cpp | 7 ++++--- cpp/src/db/db_meta_impl.h | 7 ++++--- 5 files changed, 35 insertions(+), 11 deletions(-) diff --git a/cpp/src/db/db.h b/cpp/src/db/db.h index 9707014e0f..60323d3909 100644 --- a/cpp/src/db/db.h +++ b/cpp/src/db/db.h @@ -3,6 +3,8 @@ #include #include "options.h" +#include "db_meta.h" +#include "status.h" namespace zilliz { namespace vecwise { @@ -14,8 +16,23 @@ class DB { public: static DB* Open(const Options& options_, const std::string& name_); - virtual std::string add_group(GroupOptions options_, - const std::string& group_id_) = 0; + virtual Status add_group(GroupOptions options_, + const std::string& group_id_, + GroupSchema& group_info_) = 0; + virtual Status get_group(const std::string& group_id_, GroupSchema& group_info_) = 0; + virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; + + virtual Status add_group_file(const std::string& group_id_, + GroupFileSchema& group_file_info_) = 0; + virtual Status has_group_file(const std::string& group_id_, + const std::string& file_id_, + bool& has_or_not_) = 0; + virtual Status get_group_file(const std::string& group_id_, + const std::string& file_id_, + GroupFileSchema& group_file_info_) = 0; + + virtual Status get_group_files(const std::string& group_id_, + GroupFilesSchema& group_files_info_) = 0; DB() = default; DB(const DB&) = delete; diff --git a/cpp/src/db/db_impl.h b/cpp/src/db/db_impl.h index a502c0e828..b0de407d6b 100644 --- a/cpp/src/db/db_impl.h +++ b/cpp/src/db/db_impl.h @@ -3,7 +3,9 @@ #include #include +#include #include "db.h" +#include "memvectors.h" namespace zilliz { namespace vecwise { @@ -40,6 +42,8 @@ private: bool _bg_compaction_scheduled; Status _bg_error; + MemManager _memMgr; + }; // DBImpl } // namespace engine diff --git a/cpp/src/db/db_meta.h b/cpp/src/db/db_meta.h index 7950bbfa16..10868c44f1 100644 --- a/cpp/src/db/db_meta.h +++ b/cpp/src/db/db_meta.h @@ -33,7 +33,9 @@ typedef std::vector GroupFilesSchema; class Meta { public: - virtual Status add_group(const std::string& group_id_, GroupSchema& group_info_) = 0; + virtual Status add_group(const GroupOptions& options_, + const std::string& group_id_, + GroupSchema& group_info_) = 0; virtual Status get_group(const std::string& group_id_, GroupSchema& group_info_) = 0; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; @@ -45,8 +47,7 @@ public: virtual Status get_group_file(const std::string& group_id_, const std::string& file_id_, GroupFileSchema& group_file_info_) = 0; - virtual Status mark_group_file_as_index(const std::string& group_id_, - const std::string& file_id_) = 0; + virtual Status update_group_file(const GroupFileSchema& group_file_) = 0; virtual Status get_group_files(const std::string& group_id_, GroupFilesSchema& group_files_info_) = 0; diff --git a/cpp/src/db/db_meta_impl.cpp b/cpp/src/db/db_meta_impl.cpp index 6acbc6aa81..6eace00eb1 100644 --- a/cpp/src/db/db_meta_impl.cpp +++ b/cpp/src/db/db_meta_impl.cpp @@ -14,7 +14,9 @@ Status DBMetaImpl::initialize() { return Status.OK(); } -Status DBMetaImpl::add_group(const std::string& group_id_, GroupSchema& group_info_) { +Status DBMetaImpl::add_group(const GroupOptions& options_, + const std::string& group_id_, + GroupSchema& group_info_) { //PXU TODO return Status.OK(); } @@ -55,8 +57,7 @@ Status DBMetaImpl::get_group_files(const std::string& group_id_, return Status.OK(); } -Status DBMetaImpl::mark_group_file_as_index(const std::string& group_id_, - const std::string& file_id_) { +Status DBMetaImpl::update_group_file(const GroupFileSchema& group_file_) { //PXU TODO return Status.OK(); } diff --git a/cpp/src/db/db_meta_impl.h b/cpp/src/db/db_meta_impl.h index f373a38ca9..b0745ec935 100644 --- a/cpp/src/db/db_meta_impl.h +++ b/cpp/src/db/db_meta_impl.h @@ -12,7 +12,9 @@ class DBMetaImpl : public Meta { public: DBMetaImpl(DBMetaOptions& options_); - virtual Status add_group(const std::string& group_id_, GroupSchema& group_info_) override; + virtual Status add_group(const GroupOptions& options_, + const std::string& group_id_, + GroupSchema& group_info_) override; virtual Status get_group(const std::string& group_id_, GroupSchema& group_info_) override; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; @@ -24,8 +26,7 @@ public: virtual Status get_group_file(const std::string& group_id_, const std::string& file_id_, GroupFileSchema& group_file_info_) override; - virtual Status mark_group_file_as_index(const std::string& group_id_, - const std::string& file_id_) override; + virtual Status update_group_file(const GroupFileSchema& group_file_) override; virtual Status get_group_files(const std::string& group_id_, GroupFilesSchema& group_files_info_) override; From 0d176701fbdd4b45701e220251b94e9e0b509784 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Sun, 14 Apr 2019 20:04:41 +0800 Subject: [PATCH 20/34] feat(cpp/db): add meta related method in db api Former-commit-id: b1fd80b76ae93535dacc049d9ec3ce909fa73b28 --- cpp/src/db/db.h | 11 +---------- cpp/src/db/db_impl.cpp | 21 +++++++++++++++++++-- cpp/src/db/db_impl.h | 9 ++++++++- cpp/src/db/db_meta.h | 1 + cpp/src/db/db_meta_impl.cpp | 1 + cpp/src/db/db_meta_impl.h | 1 + cpp/src/db/options.h | 4 ++++ 7 files changed, 35 insertions(+), 13 deletions(-) diff --git a/cpp/src/db/db.h b/cpp/src/db/db.h index 60323d3909..525a661e1e 100644 --- a/cpp/src/db/db.h +++ b/cpp/src/db/db.h @@ -21,17 +21,8 @@ public: GroupSchema& group_info_) = 0; virtual Status get_group(const std::string& group_id_, GroupSchema& group_info_) = 0; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; - - virtual Status add_group_file(const std::string& group_id_, - GroupFileSchema& group_file_info_) = 0; - virtual Status has_group_file(const std::string& group_id_, - const std::string& file_id_, - bool& has_or_not_) = 0; - virtual Status get_group_file(const std::string& group_id_, - const std::string& file_id_, - GroupFileSchema& group_file_info_) = 0; - virtual Status get_group_files(const std::string& group_id_, + const int date_delta_, GroupFilesSchema& group_files_info_) = 0; DB() = default; diff --git a/cpp/src/db/db_impl.cpp b/cpp/src/db/db_impl.cpp index 3fdbdf01e0..332942073c 100644 --- a/cpp/src/db/db_impl.cpp +++ b/cpp/src/db/db_impl.cpp @@ -8,15 +8,32 @@ DBImpl::DBImpl(const Options& options_, const std::string& name_) _env(options_.env), _options(options_), _bg_work_finish_signal(_mutex), - _bg_compaction_scheduled(false) { + _bg_compaction_scheduled(false), + _pMeta(new DBMetaImpl(*(_options.pMetaOptions))) { } Status DBImpl::add_group(const GroupOptions& options_, const std::string& group_id_, - std::string& gid_) { + GroupSchema& group_info_) { assert((!options_.has_id) || (options_.has_id && ("" != group_id_))); + return _pMeta->add_group(options_, group_id, group_info_); +} + +Status DBImpl::get_group(const std::string& group_id_, GroupSchema& group_info_) { + return _pMeta->get_group(group_id_, group_info_); +} + +Status DBImpl::has_group(const std::string& group_id_, bool& has_or_not_) { + return _pMeta->has_group(group_id_, has_or_not_); +} + +Status DBImpl::get_group_files(const std::string& group_id_, + const int date_delta_, + GroupFilesSchema& group_files_info_) { + return _pMeta->get_group_files(group_id_, date_delta_, group_file_info_); + } void DBImpl::try_schedule_compaction() { diff --git a/cpp/src/db/db_impl.h b/cpp/src/db/db_impl.h index b0de407d6b..f93a14e798 100644 --- a/cpp/src/db/db_impl.h +++ b/cpp/src/db/db_impl.h @@ -19,7 +19,13 @@ public: virtual Status add_group(GroupOptions options_, const std::string& group_id_, - std::string& gid_) override; + GroupSchema& group_info_) override; + virtual Status get_group(const std::string& group_id_, GroupSchema& group_info_) override; + virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; + + virtual Status get_group_files(const std::string& group_id_, + const int date_delta_, + GroupFilesSchema& group_files_info_) override; void try_schedule_compaction(); @@ -43,6 +49,7 @@ private: Status _bg_error; MemManager _memMgr; + std::shared_ptr _pMeta; }; // DBImpl diff --git a/cpp/src/db/db_meta.h b/cpp/src/db/db_meta.h index 10868c44f1..0570c44eba 100644 --- a/cpp/src/db/db_meta.h +++ b/cpp/src/db/db_meta.h @@ -50,6 +50,7 @@ public: virtual Status update_group_file(const GroupFileSchema& group_file_) = 0; virtual Status get_group_files(const std::string& group_id_, + const int date_delta_, GroupFilesSchema& group_files_info_) = 0; }; // MetaData diff --git a/cpp/src/db/db_meta_impl.cpp b/cpp/src/db/db_meta_impl.cpp index 6eace00eb1..61562378f9 100644 --- a/cpp/src/db/db_meta_impl.cpp +++ b/cpp/src/db/db_meta_impl.cpp @@ -52,6 +52,7 @@ Status DBMetaImpl::get_group_file(const std::string& group_id_, } Status DBMetaImpl::get_group_files(const std::string& group_id_, + const int date_delta_, GroupFilesSchema& group_files_info_) { // PXU TODO return Status.OK(); diff --git a/cpp/src/db/db_meta_impl.h b/cpp/src/db/db_meta_impl.h index b0745ec935..690fadf92d 100644 --- a/cpp/src/db/db_meta_impl.h +++ b/cpp/src/db/db_meta_impl.h @@ -29,6 +29,7 @@ public: virtual Status update_group_file(const GroupFileSchema& group_file_) override; virtual Status get_group_files(const std::string& group_id_, + const int date_delta_, GroupFilesSchema& group_files_info_) override; private: diff --git a/cpp/src/db/options.h b/cpp/src/db/options.h index 283c1905fc..7da3e3ce8b 100644 --- a/cpp/src/db/options.h +++ b/cpp/src/db/options.h @@ -2,15 +2,19 @@ #define VECENGINE_OPTIONS_H_ #include +#include namespace zilliz { namespace vecwise { namespace engine { +class MetaOptions; + struct Options { uint16_t memory_sync_interval = 10; uint16_t raw_file_merge_trigger_number = 100; size_t raw_to_index_trigger_size = 100000; + std::shared_ptr pMetaOptions; }; // Options From 41f85d341c88a00a9f60556a8cf79305aaac7970 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Sun, 14 Apr 2019 20:06:36 +0800 Subject: [PATCH 21/34] refactor(cpp/db): remove dummy apis Former-commit-id: ae045a47ba22ed7ca5084700359e508ce5070491 --- cpp/src/db/db_impl.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/cpp/src/db/db_impl.h b/cpp/src/db/db_impl.h index f93a14e798..cc52a05dff 100644 --- a/cpp/src/db/db_impl.h +++ b/cpp/src/db/db_impl.h @@ -36,9 +36,6 @@ private: void background_call(); void background_compaction(); - Status meta_add_group(const std::string& group_id_); - Status meta_add_group_file(const std::string& group_id_); - const _dbname; Env* const _env; const Options _options; From 79ce07c15c7323eff41c5ca6641a25db4664819d Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Sun, 14 Apr 2019 21:02:02 +0800 Subject: [PATCH 22/34] feat(cpp/db): update for memory and meta operations Former-commit-id: 69a8c7b6b9656532d4908013f8158d848481a8e6 --- cpp/src/db/db.h | 3 +++ cpp/src/db/db_impl.cpp | 8 ++++++- cpp/src/db/db_impl.h | 9 ++++++-- cpp/src/db/db_meta.h | 1 + cpp/src/db/id_generators.h | 5 +---- cpp/src/db/memvectors.cpp | 46 +++++++++++++++++++++----------------- cpp/src/db/memvectors.h | 14 ++++++++---- cpp/src/db/options.h | 1 + cpp/src/db/status.h | 6 +++++ cpp/src/db/types.h | 19 ++++++++++++++++ 10 files changed, 81 insertions(+), 31 deletions(-) create mode 100644 cpp/src/db/types.h diff --git a/cpp/src/db/db.h b/cpp/src/db/db.h index 525a661e1e..d55530408a 100644 --- a/cpp/src/db/db.h +++ b/cpp/src/db/db.h @@ -25,6 +25,9 @@ public: const int date_delta_, GroupFilesSchema& group_files_info_) = 0; + virtual Status add_vectors(const std::string& group_id_, + size_t n, const float* vectors) = 0; + DB() = default; DB(const DB&) = delete; DB& operator=(const DB&) = delete; diff --git a/cpp/src/db/db_impl.cpp b/cpp/src/db/db_impl.cpp index 332942073c..b5c85b1c20 100644 --- a/cpp/src/db/db_impl.cpp +++ b/cpp/src/db/db_impl.cpp @@ -9,7 +9,8 @@ DBImpl::DBImpl(const Options& options_, const std::string& name_) _options(options_), _bg_work_finish_signal(_mutex), _bg_compaction_scheduled(false), - _pMeta(new DBMetaImpl(*(_options.pMetaOptions))) { + _pMeta(new DBMetaImpl(*(_options.pMetaOptions))), + _pMemMgr(new MemManager(_pMeta)) { } Status DBImpl::add_group(const GroupOptions& options_, @@ -36,6 +37,11 @@ Status DBImpl::get_group_files(const std::string& group_id_, } +Status DBImpl::add_vectors(const std::string& group_id_, + size_t n, const float* vectors, IDNumbers& vector_ids_) { + return _pMemMgr->add_vectors(group_id_, n, vectors, vector_ids_); +} + void DBImpl::try_schedule_compaction() { if (_bg_compaction_scheduled) return; if (!_bg_error.ok()) return; diff --git a/cpp/src/db/db_impl.h b/cpp/src/db/db_impl.h index cc52a05dff..03fc3d3c0b 100644 --- a/cpp/src/db/db_impl.h +++ b/cpp/src/db/db_impl.h @@ -6,6 +6,7 @@ #include #include "db.h" #include "memvectors.h" +#include "types.h" namespace zilliz { namespace vecwise { @@ -27,11 +28,15 @@ public: const int date_delta_, GroupFilesSchema& group_files_info_) override; - void try_schedule_compaction(); + virtual Status add_vectors(const std::string& group_id_, + size_t n, const float* vectors, IDNumbers& vector_ids_) override; virtual ~DBImpl(); + private: + void try_schedule_compaction(); + static void BGWork(void* db); void background_call(); void background_compaction(); @@ -45,8 +50,8 @@ private: bool _bg_compaction_scheduled; Status _bg_error; - MemManager _memMgr; std::shared_ptr _pMeta; + std::shared_ptr _pMemMgr; }; // DBImpl diff --git a/cpp/src/db/db_meta.h b/cpp/src/db/db_meta.h index 0570c44eba..da2b40edf0 100644 --- a/cpp/src/db/db_meta.h +++ b/cpp/src/db/db_meta.h @@ -11,6 +11,7 @@ struct GroupSchema { size_t files_cnt = 0; uint16_t dimension; std::string location = ""; + std::string next_file_location = ""; }; // GroupSchema diff --git a/cpp/src/db/id_generators.h b/cpp/src/db/id_generators.h index 767c3ce322..02f3915bcc 100644 --- a/cpp/src/db/id_generators.h +++ b/cpp/src/db/id_generators.h @@ -2,15 +2,12 @@ #define UTILS_ID_GENERATORS_H_ #include +#include "types.h" namespace zilliz { namespace vecwise { namespace engine { -#define uint64_t IDNumber; -#define IDNumber* IDNumberPtr; -#define std::vector IDNumbers; - class IDGenerator { public: virtual IDNumber getNextIDNumber() = 0; diff --git a/cpp/src/db/memvectors.cpp b/cpp/src/db/memvectors.cpp index 3a30dfda9e..f546fcf02b 100644 --- a/cpp/src/db/memvectors.cpp +++ b/cpp/src/db/memvectors.cpp @@ -3,6 +3,7 @@ #include #include "memvectors.h" +#include "db_meta.h" namespace vecengine { @@ -15,10 +16,9 @@ MemVectors::MemVectors(size_t dimension_, const std::string& file_location_) : _pIdMapIndex = new faiss::IndexIDMap(_pInnerIndex) { } -IDNumbers&& MemVectors::add(size_t n, const float* vectors) { - IDNumbers&& ids = _pIdGenerator->getNextIDNumbers(n); - _pIdMapIndex->add_with_ids(n, vectors, pIds, &ids[0]); - return ids; +void MemVectors::add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { + vector_ids_ = _pIdGenerator->getNextIDNumbers(n_); + _pIdMapIndex->add_with_ids(n_, vectors_, &vector_ids_[0]); } size_t MemVectors::total() const { @@ -57,27 +57,33 @@ MemVectors* MemManager::get_mem_by_group(const std::string& group_id_) { if memIt != _memMap.end() { return &(memIt->second); } - // PXU TODO: - // 1. Read Group meta info - // 2. Initalize MemVectors base meta info - return nullptr; - /* GroupMetaInfo info; */ - /* bool succ = env->getGroupMeta(group_id, &info); */ - /* if (!succ) { */ - /* return nullptr; */ - /* } */ - /* _memMap[group_id] = MemVectors(info.dimension, info.next_file_location); */ - /* return &(_memMap[group_id]); */ + + GroupSchema group_info; + Status status = _pMeta->get_group(group_id_, group_info); + if (!status.ok()) { + return nullptr; + } + _memMap[group_id] = MemVectors(group_info.dimension, group_info.next_file_location); + return &(_memMap[group_id]); } -IDNumbers&& MemManager::add_vectors_no_lock(const std::string& group_id_, +Status MemManager::add_vectors(const std::string& group_id_, + size_t n_, + const float* vectors_, + IDNumbers& vector_ids_) { + // PXU TODO + return add_vectors_no_lock(group_id_, n_, vectors_, vector_ids_); +} + +Status MemManager::add_vectors_no_lock(const std::string& group_id_, size_t n, - const float* vectors) { - auto mem = get_group_mem(group_id_); + const float* vectors, + IDNumbers& vector_ids_) { + auto mem = get_mem_by_group(group_id_); if (mem == nullptr) { - return IDNumbers(); + return Status::NotFound("Group " + group_id_ " not found!"); } - return mem->add(n, vectors); + return mem->add(n, vectors, vector_ids_); } diff --git a/cpp/src/db/memvectors.h b/cpp/src/db/memvectors.h index 159ee441d4..a6741971a1 100644 --- a/cpp/src/db/memvectors.h +++ b/cpp/src/db/memvectors.h @@ -4,6 +4,7 @@ #include #include #include "id_generators.h" +#include "status.h" class faiss::IndexIDMap; class faiss::Index; @@ -37,19 +38,24 @@ private: }; // MemVectors +class Meta; + class MemManager { public: - MemManager() = default; + MemManager(const std::shared_ptr& meta_) : _pMeta(meta_) {} MemVectors* get_mem_by_group(const std::string& group_id_); + Status add_vectors(const std::string& group_id_, + size_t n_, const float* vectors_, IDNumbers& vector_ids_); + private: - IDNumbers&& add_vectors_no_lock(const std::string& group_id_, - size_t n, - const float* vectors); + Status add_vectors_no_lock(const std::string& group_id_, + size_t n_, const float* vectors_, IDNumbers& vector_ids_); typedef std::map MemMap; MemMap _memMap; + std::shared_ptr _pMeta; }; // MemManager diff --git a/cpp/src/db/options.h b/cpp/src/db/options.h index 7da3e3ce8b..cd0ceb89a7 100644 --- a/cpp/src/db/options.h +++ b/cpp/src/db/options.h @@ -9,6 +9,7 @@ namespace vecwise { namespace engine { class MetaOptions; +class Env; struct Options { uint16_t memory_sync_interval = 10; diff --git a/cpp/src/db/status.h b/cpp/src/db/status.h index 3133c64924..85ac57c1c8 100644 --- a/cpp/src/db/status.h +++ b/cpp/src/db/status.h @@ -17,14 +17,20 @@ public: Status& operator=(const Status& rhs_) noexcept; static Status OK() { return Status(); } + static Status NotFound(const std::string& msg_, const std::string& msg2_="") { + return Status(kNotFound, msg_, msg2_); + } bool ok() const { return _state == nullptr; } + bool IsNotFound() const { return code() == kNotFound; } + private: const char* _state; enum Code { kOK = 0, + kNotFound, }; Code code() const { diff --git a/cpp/src/db/types.h b/cpp/src/db/types.h new file mode 100644 index 0000000000..185fb73641 --- /dev/null +++ b/cpp/src/db/types.h @@ -0,0 +1,19 @@ +#ifndef VECENGINE_TYPES_H_ +#define VECENGINE_TYPES_H_ + +#include + +namespace zilliz { +namespace vecwise { +namespace engine { + +#define uint64_t IDNumber; +#define IDNumber* IDNumberPtr; +#define std::vector IDNumbers; + + +} // namespace engine +} // namespace vecwise +} // namespace zilliz + +#endif // VECENGINE_TYPES_H_ From 33d96b02b1e961cb320a19db1fb78b4310c5c2d1 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 15 Apr 2019 12:20:54 +0800 Subject: [PATCH 23/34] feat(cpp/db): add timer thread Former-commit-id: 47f932b6d49c14b11e3ed69dbfbf0de840f6d98d --- cpp/src/db/db_impl.cpp | 38 ++++++++++++++++++++++++++++++++++---- cpp/src/db/db_impl.h | 1 + cpp/src/db/memvectors.cpp | 39 +++++++++++++++++++++++++++++++++++---- cpp/src/db/memvectors.h | 15 ++++++++++++--- 4 files changed, 82 insertions(+), 11 deletions(-) diff --git a/cpp/src/db/db_impl.cpp b/cpp/src/db/db_impl.cpp index b5c85b1c20..7f7e19a748 100644 --- a/cpp/src/db/db_impl.cpp +++ b/cpp/src/db/db_impl.cpp @@ -1,4 +1,5 @@ #include +#include #include "db_impl.h" namespace vecengine { @@ -9,8 +10,10 @@ DBImpl::DBImpl(const Options& options_, const std::string& name_) _options(options_), _bg_work_finish_signal(_mutex), _bg_compaction_scheduled(false), + _shutting_down(false), _pMeta(new DBMetaImpl(*(_options.pMetaOptions))), _pMemMgr(new MemManager(_pMeta)) { + start_timer_task(Options.memory_sync_interval); } Status DBImpl::add_group(const GroupOptions& options_, @@ -39,7 +42,27 @@ Status DBImpl::get_group_files(const std::string& group_id_, Status DBImpl::add_vectors(const std::string& group_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) { - return _pMemMgr->add_vectors(group_id_, n, vectors, vector_ids_); + Status status = _pMemMgr->add_vectors(group_id_, n, vectors, vector_ids_); + if (!status.ok()) { + return status; + } +} + +void DBImpl::start_timer_task(int interval_) { + std::thread bg_task(&DBImpl::background_timer_task, this, interval_); + bg_task.detach(); +} + +void DBImpl::background_timer_task(int interval_) { + Status status; + while (true) { + if (!_bg_error.ok()) break; + if (_shutting_down.load(std::memory_order_acquire)) break; + + std::this_thread::sleep_for(std::chrono::seconds(interval_)); + + try_schedule_compaction(); + } } void DBImpl::try_schedule_compaction() { @@ -61,14 +84,21 @@ void DBImpl::background_call() { if (!_bg_error.ok()) return; background_compaction(); + + _bg_compaction_scheduled = false; + _bg_work_finish_signal.notify_all(); } void DBImpl::background_compaction() { - + _pMemMgr->serialize(); } -void DBImpl::compact_memory() { - +DBImpl::~DBImpl() { + std::lock_guard _mutex; + _shutting_down.store(true, std::memory_order_release); + while (_bg_compaction_scheduled) { + _bg_work_finish_signal.wait(); + } } /* diff --git a/cpp/src/db/db_impl.h b/cpp/src/db/db_impl.h index 03fc3d3c0b..70ab2c5a2e 100644 --- a/cpp/src/db/db_impl.h +++ b/cpp/src/db/db_impl.h @@ -49,6 +49,7 @@ private: std::condition_variable _bg_work_finish_signal; bool _bg_compaction_scheduled; Status _bg_error; + std::atomic _shutting_down; std::shared_ptr _pMeta; std::shared_ptr _pMemMgr; diff --git a/cpp/src/db/memvectors.cpp b/cpp/src/db/memvectors.cpp index f546fcf02b..f76589a628 100644 --- a/cpp/src/db/memvectors.cpp +++ b/cpp/src/db/memvectors.cpp @@ -52,7 +52,7 @@ MemVectors::~MemVectors() { * MemManager */ -MemVectors* MemManager::get_mem_by_group(const std::string& group_id_) { +VectorsPtr MemManager::get_mem_by_group(const std::string& group_id_) { auto memIt = _memMap.find(group_id_); if memIt != _memMap.end() { return &(memIt->second); @@ -63,15 +63,16 @@ MemVectors* MemManager::get_mem_by_group(const std::string& group_id_) { if (!status.ok()) { return nullptr; } - _memMap[group_id] = MemVectors(group_info.dimension, group_info.next_file_location); - return &(_memMap[group_id]); + _memMap[group_id] = std::shared_ptr(new MemVectors(group_info.dimension, + group_info.next_file_location)); + return _memMap[group_id]; } Status MemManager::add_vectors(const std::string& group_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_) { - // PXU TODO + std::lock_guard lock(_mutex); return add_vectors_no_lock(group_id_, n_, vectors_, vector_ids_); } @@ -86,5 +87,35 @@ Status MemManager::add_vectors_no_lock(const std::string& group_id_, return mem->add(n, vectors, vector_ids_); } +Status MemManager::mark_memory_as_immutable() { + std::lock_guard lock(_mutex); + for (auto& kv: _memMap) { + _immMems.push_back(kv.second); + } + _memMap.clear(); +} + +/* bool MemManager::need_serialize(double interval) { */ +/* if (_immMems.size() > 0) { */ +/* return false; */ +/* } */ + +/* auto diff = std::difftime(std::time(nullptr), _last_compact_time); */ +/* if (diff >= interval) { */ +/* return true; */ +/* } */ + +/* return false; */ +/* } */ + +Status MemManager::serialize() { + mark_memory_as_immutable(); + for (auto& mem : _immMems) { + mem->serialize() + } + _immMems.clear(); + /* _last_compact_time = std::time(nullptr); */ +} + } // namespace vecengine diff --git a/cpp/src/db/memvectors.h b/cpp/src/db/memvectors.h index a6741971a1..aff162a15e 100644 --- a/cpp/src/db/memvectors.h +++ b/cpp/src/db/memvectors.h @@ -3,6 +3,7 @@ #include #include +#include #include "id_generators.h" #include "status.h" @@ -42,20 +43,28 @@ class Meta; class MemManager { public: - MemManager(const std::shared_ptr& meta_) : _pMeta(meta_) {} + typedef std::shared_ptr VectorsPtr; + MemManager(const std::shared_ptr& meta_) + : _pMeta(meta_), _last_compact_time(std::time(nullptr)) {} - MemVectors* get_mem_by_group(const std::string& group_id_); + VectorsPtr get_mem_by_group(const std::string& group_id_); Status add_vectors(const std::string& group_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_); + Status serialize(); + private: Status add_vectors_no_lock(const std::string& group_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_); - typedef std::map MemMap; + typedef std::map MemMap; + typedef std::vector ImmMemPool; MemMap _memMap; + ImmMemPool _immMems; std::shared_ptr _pMeta; + std::time_t _last_compact_time; + std::mutex _mutex; }; // MemManager From f0a66cac521b5b62c17c7dd711d7a0372233b27e Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 15 Apr 2019 13:32:03 +0800 Subject: [PATCH 24/34] fix(db): fix status compile error Former-commit-id: be4be51b0e1e456db2f12a809dddf2b293ece40f --- cpp/src/db/status.cpp | 40 ++++++++++++++++++++++-------------- cpp/src/db/status.h | 47 ++++++++++++++++++++++--------------------- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/cpp/src/db/status.cpp b/cpp/src/db/status.cpp index 8c547d5abe..457565d10c 100644 --- a/cpp/src/db/status.cpp +++ b/cpp/src/db/status.cpp @@ -1,40 +1,48 @@ +#include +#include +#include #include "status.h" -namespace vecengine { +namespace zilliz { +namespace vecwise { +namespace engine { -const char* Status::CopyState(const char* state_) { +const char* Status::CopyState(const char* state) { uint32_t size; - memcpy(&size, state_, sizeof(size)); - char result = new char[size+5]; - memcpy(result, state_, size+5); + std::memcpy(&size, state, sizeof(size)); + char* result = new char[size+5]; + memcpy(result, state, size+5); return result; } -Status::Status(Code code_, const std::string& msg_, const std::string& msg2_) { - assert(code_ != kOK); - const uint32_t len1 = msg_.size(); - const uint32_t len2 = msg2_.size(); +Status::Status(Code code, const std::string& msg, const std::string& msg2) { + assert(code != kOK); + const uint32_t len1 = msg.size(); + const uint32_t len2 = msg2.size(); const uint32_t size = len1 + (len2 ? (2+len2) : 0); char* result = new char[size+5]; - memcpy(result, &size, sizeof(size)); + std::memcpy(result, &size, sizeof(size)); result[4] = static_cast(code); - memcpy(result+5, msg_.data(), len1); + memcpy(result+5, msg.data(), len1); if (len2) { result[5 + len1] = ':'; result[6 + len1] = ' '; - memcpy(result + 7 + len1, msg2_.data(), len2); + memcpy(result + 7 + len1, msg2.data(), len2); } - _state = result; + state_ = result; } std::string Status::ToString() const { - if (_state == nullptr) return "OK"; + if (state_ == nullptr) return "OK"; char tmp[30]; const char* type; switch (code()) { case kOK: type = "OK"; break; + case kNotFound: + type = "NotFound: "; + break; default: snprintf(tmp, sizeof(tmp), "Unkown code(%d): ", static_cast(code())); @@ -49,4 +57,6 @@ std::string Status::ToString() const { return result; } -} // namespace vecengine +} // namespace engine +} // namespace vecwise +} // namespace zilliz diff --git a/cpp/src/db/status.h b/cpp/src/db/status.h index 85ac57c1c8..80be027c9b 100644 --- a/cpp/src/db/status.h +++ b/cpp/src/db/status.h @@ -1,5 +1,6 @@ -#ifndef VECENGINE_STATUS_H_ -#define VECENGINE_STATUS_H_ +#pragma once + +#include namespace zilliz { namespace vecwise { @@ -7,26 +8,28 @@ namespace engine { class Status { public: - Status() noexcept : _state(nullptr) {} - ~Status() { delete[] _state; } + Status() noexcept : state_(nullptr) {} + ~Status() { delete[] state_; } - Status(const Status& rhs_); - Status& operator=(const Status& rhs_); + Status(const Status& rhs); + Status& operator=(const Status& rhs); - Status(const Status&& rhs_) noexcept : _state(rhs_._state) { rhs_._state = nullptr; } - Status& operator=(const Status& rhs_) noexcept; + Status(Status&& rhs) noexcept : state_(rhs.state_) { rhs.state_ = nullptr; } + Status& operator=(Status&& rhs_) noexcept; static Status OK() { return Status(); } - static Status NotFound(const std::string& msg_, const std::string& msg2_="") { - return Status(kNotFound, msg_, msg2_); + static Status NotFound(const std::string& msg, const std::string& msg2="") { + return Status(kNotFound, msg, msg2); } - bool ok() const { return _state == nullptr; } + bool ok() const { return state_ == nullptr; } bool IsNotFound() const { return code() == kNotFound; } + std::string ToString() const; + private: - const char* _state; + const char* state_; enum Code { kOK = 0, @@ -34,32 +37,30 @@ private: }; Code code() const { - return (_state == nullptr) ? kOK : static_cast(_state[4]) + return (state_ == nullptr) ? kOK : static_cast(state_[4]); } - + Status(Code code, const std::string& msg, const std::string& msg2); static const char* CopyState(const char* s); }; // Status -inline Status::Status(const Status* rhs_) { - _state = (rhs_._state == nullptr) ? nullptr : CopyState(rhs_._state); +inline Status::Status(const Status& rhs) { + state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_); } -inline Status& Status::operator=(const Status& rhs_) { - if (_state != rhs_._state) { +inline Status& Status::operator=(const Status& rhs) { + if (state_ != rhs.state_) { delete[] state_; - _state = (rhs_._state == nullptr) ? nullptr : CopyState(rhs_._state); + state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_); } return *this; } -inline Status& Status::operator=(Status&& rhs_) noexcept { - std::swap(_state, rhs_._state); +inline Status& Status::operator=(Status&& rhs) noexcept { + std::swap(state_, rhs.state_); return *this; } } // namespace engine } // namespace vecwise } // namespace zilliz - -#endif // VECENGINE_STATUS_H_ From 89125a9fb66b9c108bd6abe1c768645fb510a1b9 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 15 Apr 2019 13:43:28 +0800 Subject: [PATCH 25/34] fix(db): fix id generator errors Former-commit-id: 898a8d587fe81717bc51aba9d5672775b819f100 --- cpp/src/db/id_generators.cpp | 25 +++++++++++++++---------- cpp/src/db/id_generators.h | 7 ++----- cpp/src/db/types.h | 11 ++++------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/cpp/src/db/id_generators.cpp b/cpp/src/db/id_generators.cpp index 8296a4048f..3541048413 100644 --- a/cpp/src/db/id_generators.cpp +++ b/cpp/src/db/id_generators.cpp @@ -1,32 +1,37 @@ #include #include -#inlcude "id_generators.h" +#include "id_generators.h" -using std::chrono; -namespace vecengine { +namespace zilliz { +namespace vecwise { +namespace engine { IDGenerator::~IDGenerator() {} IDNumber SimpleIDGenerator::getNextIDNumber() { - auto now = chrono::system_clock::now(); - auto micros = duration_cast(now.time_since_epoch()).count(); - return micros * MAX_IDS_PER_MICRO + auto now = std::chrono::system_clock::now(); + auto micros = std::chrono::duration_cast( + now.time_since_epoch()).count(); + return micros * MAX_IDS_PER_MICRO; } IDNumbers&& SimpleIDGenerator::getNextIDNumbers(size_t n) { assert(n < MAX_IDS_PER_MICRO); - auto now = chrono::system_clock::now(); - auto micros = duration_cast(now.time_since_epoch()).count(); + auto now = std::chrono::system_clock::now(); + auto micros = std::chrono::duration_cast( + now.time_since_epoch()).count(); micros *= MAX_IDS_PER_MICRO; IDNumbers ids = IDNumbers(n); for (int pos=0; pos #include "types.h" @@ -24,7 +23,7 @@ public: virtual IDNumbers&& getNextIDNumbers(size_t n_) override; private: - const MAX_IDS_PER_MICRO = 1000; + const size_t MAX_IDS_PER_MICRO = 1000; }; // SimpleIDGenerator @@ -32,5 +31,3 @@ private: } // namespace engine } // namespace vecwise } // namespace zilliz - -#endif // UTILS_ID_GENERATORS_H_ diff --git a/cpp/src/db/types.h b/cpp/src/db/types.h index 185fb73641..7fd33a0787 100644 --- a/cpp/src/db/types.h +++ b/cpp/src/db/types.h @@ -1,5 +1,4 @@ -#ifndef VECENGINE_TYPES_H_ -#define VECENGINE_TYPES_H_ +#pragma once #include @@ -7,13 +6,11 @@ namespace zilliz { namespace vecwise { namespace engine { -#define uint64_t IDNumber; -#define IDNumber* IDNumberPtr; -#define std::vector IDNumbers; +typedef uint64_t IDNumber; +typedef IDNumber* IDNumberPtr; +typedef std::vector IDNumbers; } // namespace engine } // namespace vecwise } // namespace zilliz - -#endif // VECENGINE_TYPES_H_ From ea1241dc21494b7a6e1112e94b6bb7fd93ca8889 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 15 Apr 2019 13:54:10 +0800 Subject: [PATCH 26/34] fix(db): fix env compile error Former-commit-id: aaf0f190a14e88cdfeb69e6ededc14f739181841 --- cpp/src/db/env.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cpp/src/db/env.cpp b/cpp/src/db/env.cpp index 8b1a288f9f..760a7e4323 100644 --- a/cpp/src/db/env.cpp +++ b/cpp/src/db/env.cpp @@ -1,16 +1,15 @@ +#include #include "env.h" namespace zilliz { namespace vecwise { namespace engine { -Env::Env() - : _bg_work_cv(&_bg_work_mutex), - _bg_work_started(false) { +Env::Env() : _bg_work_started(false) { } void Env::schedule(void (*function_)(void* arg_), void* arg_) { - std::lock_guard lock; + std::unique_lock lock(_bg_work_mutex); if (!_bg_work_started) { _bg_work_started = true; @@ -27,9 +26,9 @@ void Env::schedule(void (*function_)(void* arg_), void* arg_) { void Env::backgroud_thread_main() { while (true) { - std::lock_guard lock; + std::unique_lock lock(_bg_work_mutex); while (_bg_work_queue.empty()) { - _bg_work_cv.wait(); + _bg_work_cv.wait(lock); } assert(!_bg_work_queue.empty()); From 94c7204d7ca7c563885a4ae71563db05d1d8c080 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 15 Apr 2019 14:07:05 +0800 Subject: [PATCH 27/34] fix(db): fix meta compile error Former-commit-id: 7217f5dc230517a1d0445bfb156e3ba45af094ae --- cpp/src/db/db_meta.cpp | 1 - cpp/src/db/db_meta.h | 11 +++++++---- cpp/src/db/db_meta_impl.cpp | 20 ++++++++++---------- cpp/src/db/db_meta_impl.h | 2 +- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/cpp/src/db/db_meta.cpp b/cpp/src/db/db_meta.cpp index 42c1d9d78f..9274b8258e 100644 --- a/cpp/src/db/db_meta.cpp +++ b/cpp/src/db/db_meta.cpp @@ -4,7 +4,6 @@ namespace zilliz { namespace vecwise { namespace engine { -} } // namespace engine } // namespace vecwise diff --git a/cpp/src/db/db_meta.h b/cpp/src/db/db_meta.h index da2b40edf0..1eae4b95a6 100644 --- a/cpp/src/db/db_meta.h +++ b/cpp/src/db/db_meta.h @@ -1,5 +1,10 @@ -#ifndef VECENGINE_DB_META_H_ -#define VECENGINE_DB_META_H_ +#pragma once + +#include +#include +#include +#include "options.h" +#include "status.h" namespace zilliz { namespace vecwise { @@ -59,5 +64,3 @@ public: } // namespace engine } // namespace vecwise } // namespace zilliz - -#endif // VECENGINE_DB_META_H_ diff --git a/cpp/src/db/db_meta_impl.cpp b/cpp/src/db/db_meta_impl.cpp index 61562378f9..a4ba4e810e 100644 --- a/cpp/src/db/db_meta_impl.cpp +++ b/cpp/src/db/db_meta_impl.cpp @@ -4,63 +4,63 @@ namespace zilliz { namespace vecwise { namespace engine { -Status DBMetaImpl::DBMetaImpl(DBMetaOptions options_) +DBMetaImpl::DBMetaImpl(const DBMetaOptions& options_) : _options(options_) { initialize(); } Status DBMetaImpl::initialize() { // PXU TODO: Create DB Connection - return Status.OK(); + return Status::OK(); } Status DBMetaImpl::add_group(const GroupOptions& options_, const std::string& group_id_, GroupSchema& group_info_) { //PXU TODO - return Status.OK(); + return Status::OK(); } Status DBMetaImpl::get_group(const std::string& group_id_, GroupSchema& group_info_) { //PXU TODO - return Status.OK(); + return Status::OK(); } Status DBMetaImpl::has_group(const std::string& group_id_, bool& has_or_not_) { //PXU TODO - return Status.OK(); + return Status::OK(); } Status DBMetaImpl::add_group_file(const std::string& group_id_, GroupFileSchema& group_file_info_) { //PXU TODO - return Status.OK(); + return Status::OK(); } Status DBMetaImpl::has_group_file(const std::string& group_id_, const std::string& file_id_, bool& has_or_not_) { //PXU TODO - return Status.OK(); + return Status::OK(); } Status DBMetaImpl::get_group_file(const std::string& group_id_, const std::string& file_id_, GroupFileSchema& group_file_info_) { //PXU TODO - return Status.OK(); + return Status::OK(); } Status DBMetaImpl::get_group_files(const std::string& group_id_, const int date_delta_, GroupFilesSchema& group_files_info_) { // PXU TODO - return Status.OK(); + return Status::OK(); } Status DBMetaImpl::update_group_file(const GroupFileSchema& group_file_) { //PXU TODO - return Status.OK(); + return Status::OK(); } } // namespace engine diff --git a/cpp/src/db/db_meta_impl.h b/cpp/src/db/db_meta_impl.h index 690fadf92d..bf18b713d4 100644 --- a/cpp/src/db/db_meta_impl.h +++ b/cpp/src/db/db_meta_impl.h @@ -10,7 +10,7 @@ namespace engine { class DBMetaImpl : public Meta { public: - DBMetaImpl(DBMetaOptions& options_); + DBMetaImpl(const DBMetaOptions& options_); virtual Status add_group(const GroupOptions& options_, const std::string& group_id_, From 5c5e3205f0e48b34319da4690bb5dc99e36b29ad Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 15 Apr 2019 14:56:40 +0800 Subject: [PATCH 28/34] fix(db): fix db compile errors Former-commit-id: ac0e3e3a6a6bc82dec43eb431592633ce1292db2 --- cpp/src/db/db.h | 5 +++-- cpp/src/db/db_impl.cpp | 38 +++++++++++++++++++++---------------- cpp/src/db/db_impl.h | 14 ++++++-------- cpp/src/db/db_meta_impl.cpp | 4 ++-- cpp/src/db/db_meta_impl.h | 4 ++-- cpp/src/db/env.cpp | 5 +++++ cpp/src/db/env.h | 2 ++ cpp/src/db/memvectors.h | 6 ++++-- cpp/src/db/options.cpp | 13 +++++++++++++ cpp/src/db/options.h | 1 + 10 files changed, 60 insertions(+), 32 deletions(-) create mode 100644 cpp/src/db/options.cpp diff --git a/cpp/src/db/db.h b/cpp/src/db/db.h index d55530408a..abaa498421 100644 --- a/cpp/src/db/db.h +++ b/cpp/src/db/db.h @@ -5,6 +5,7 @@ #include "options.h" #include "db_meta.h" #include "status.h" +#include "types.h" namespace zilliz { namespace vecwise { @@ -16,7 +17,7 @@ class DB { public: static DB* Open(const Options& options_, const std::string& name_); - virtual Status add_group(GroupOptions options_, + virtual Status add_group(const GroupOptions& options_, const std::string& group_id_, GroupSchema& group_info_) = 0; virtual Status get_group(const std::string& group_id_, GroupSchema& group_info_) = 0; @@ -26,7 +27,7 @@ public: GroupFilesSchema& group_files_info_) = 0; virtual Status add_vectors(const std::string& group_id_, - size_t n, const float* vectors) = 0; + size_t n, const float* vectors, IDNumbers& vector_ids_) = 0; DB() = default; DB(const DB&) = delete; diff --git a/cpp/src/db/db_impl.cpp b/cpp/src/db/db_impl.cpp index 7f7e19a748..a23fbe1db6 100644 --- a/cpp/src/db/db_impl.cpp +++ b/cpp/src/db/db_impl.cpp @@ -1,28 +1,32 @@ #include #include +#include #include "db_impl.h" +#include "db_meta_impl.h" +#include "env.h" -namespace vecengine { +namespace zilliz { +namespace vecwise { +namespace engine { DBImpl::DBImpl(const Options& options_, const std::string& name_) : _dbname(name_), _env(options_.env), _options(options_), - _bg_work_finish_signal(_mutex), _bg_compaction_scheduled(false), _shutting_down(false), _pMeta(new DBMetaImpl(*(_options.pMetaOptions))), _pMemMgr(new MemManager(_pMeta)) { - start_timer_task(Options.memory_sync_interval); + start_timer_task(options_.memory_sync_interval); } -Status DBImpl::add_group(const GroupOptions& options_, - const std::string& group_id_, - GroupSchema& group_info_) { - assert((!options_.has_id) || - (options_.has_id && ("" != group_id_))); +Status DBImpl::add_group(const GroupOptions& options, + const std::string& group_id, + GroupSchema& group_info) { + assert((!options.has_id) || + (options.has_id && ("" != group_id))); - return _pMeta->add_group(options_, group_id, group_info_); + return _pMeta->add_group(options, group_id, group_info); } Status DBImpl::get_group(const std::string& group_id_, GroupSchema& group_info_) { @@ -33,10 +37,10 @@ Status DBImpl::has_group(const std::string& group_id_, bool& has_or_not_) { return _pMeta->has_group(group_id_, has_or_not_); } -Status DBImpl::get_group_files(const std::string& group_id_, - const int date_delta_, - GroupFilesSchema& group_files_info_) { - return _pMeta->get_group_files(group_id_, date_delta_, group_file_info_); +Status DBImpl::get_group_files(const std::string& group_id, + const int date_delta, + GroupFilesSchema& group_files_info) { + return _pMeta->get_group_files(group_id, date_delta, group_files_info); } @@ -94,10 +98,10 @@ void DBImpl::background_compaction() { } DBImpl::~DBImpl() { - std::lock_guard _mutex; + std::unique_lock lock(_mutex); _shutting_down.store(true, std::memory_order_release); while (_bg_compaction_scheduled) { - _bg_work_finish_signal.wait(); + _bg_work_finish_signal.wait(lock); } } @@ -112,4 +116,6 @@ DB* DB::Open(const Options& options_, const std::string& name_) { return impl; } -} // namespace vecengine +} // namespace engine +} // namespace vecwise +} // namespace zilliz diff --git a/cpp/src/db/db_impl.h b/cpp/src/db/db_impl.h index 70ab2c5a2e..d9e3a3d7c9 100644 --- a/cpp/src/db/db_impl.h +++ b/cpp/src/db/db_impl.h @@ -1,9 +1,9 @@ -#ifndef VECENGINE_DB_IMPL_H_ -#define VECENGINE_DB_IMPL_H_ +#pragma once #include #include #include +#include #include "db.h" #include "memvectors.h" #include "types.h" @@ -18,7 +18,7 @@ class DBImpl : public DB { public: DBImpl(const Options& options_, const std::string& name_); - virtual Status add_group(GroupOptions options_, + virtual Status add_group(const GroupOptions& options_, const std::string& group_id_, GroupSchema& group_info_) override; virtual Status get_group(const std::string& group_id_, GroupSchema& group_info_) override; @@ -36,12 +36,14 @@ public: private: void try_schedule_compaction(); + void start_timer_task(int interval_); + void background_timer_task(int interval_); static void BGWork(void* db); void background_call(); void background_compaction(); - const _dbname; + const std::string& _dbname; Env* const _env; const Options _options; @@ -59,7 +61,3 @@ private: } // namespace engine } // namespace vecwise } // namespace zilliz - -#endif // VECENGINE_DB_META_IMPL_H_ - -#endif // VECENGINE_DB_IMPL_H_ diff --git a/cpp/src/db/db_meta_impl.cpp b/cpp/src/db/db_meta_impl.cpp index a4ba4e810e..23ca5dd6d2 100644 --- a/cpp/src/db/db_meta_impl.cpp +++ b/cpp/src/db/db_meta_impl.cpp @@ -4,8 +4,8 @@ namespace zilliz { namespace vecwise { namespace engine { -DBMetaImpl::DBMetaImpl(const DBMetaOptions& options_) - : _options(options_) { +DBMetaImpl::DBMetaImpl(const MetaOptions& options_) + : _options(static_cast(options_)) { initialize(); } diff --git a/cpp/src/db/db_meta_impl.h b/cpp/src/db/db_meta_impl.h index bf18b713d4..f8602c1635 100644 --- a/cpp/src/db/db_meta_impl.h +++ b/cpp/src/db/db_meta_impl.h @@ -10,7 +10,7 @@ namespace engine { class DBMetaImpl : public Meta { public: - DBMetaImpl(const DBMetaOptions& options_); + DBMetaImpl(const MetaOptions& options_); virtual Status add_group(const GroupOptions& options_, const std::string& group_id_, @@ -36,7 +36,7 @@ private: Status initialize(); - const DBMetaOptions _options; + const DBMetaOptions& _options; }; // DBMetaImpl diff --git a/cpp/src/db/env.cpp b/cpp/src/db/env.cpp index 760a7e4323..7b97cc0146 100644 --- a/cpp/src/db/env.cpp +++ b/cpp/src/db/env.cpp @@ -43,6 +43,11 @@ void Env::backgroud_thread_main() { Env::~Env() {} +Env* Env::Default() { + static Env env; + return &env; +} + } // namespace engine } // namespace vecwise } // namespace zilliz diff --git a/cpp/src/db/env.h b/cpp/src/db/env.h index 2d2b630606..727dd86c59 100644 --- a/cpp/src/db/env.h +++ b/cpp/src/db/env.h @@ -21,6 +21,8 @@ public: virtual ~Env(); + static Env* Default(); + protected: void backgroud_thread_main(); static void BackgroundThreadEntryPoint(Env* env) { diff --git a/cpp/src/db/memvectors.h b/cpp/src/db/memvectors.h index aff162a15e..034d039010 100644 --- a/cpp/src/db/memvectors.h +++ b/cpp/src/db/memvectors.h @@ -7,8 +7,10 @@ #include "id_generators.h" #include "status.h" -class faiss::IndexIDMap; -class faiss::Index; +namespace faiss { + class IndexIDMap; + class Index; +} namespace zilliz { diff --git a/cpp/src/db/options.cpp b/cpp/src/db/options.cpp new file mode 100644 index 0000000000..dd28cdb7dd --- /dev/null +++ b/cpp/src/db/options.cpp @@ -0,0 +1,13 @@ +#include "options.h" + +namespace zilliz { +namespace vecwise { +namespace engine { + +Options::Options() + : env(Env::Default()) { +} + +} // namespace engine +} // namespace vecwise +} // namespace zilliz diff --git a/cpp/src/db/options.h b/cpp/src/db/options.h index cd0ceb89a7..aefd094edf 100644 --- a/cpp/src/db/options.h +++ b/cpp/src/db/options.h @@ -16,6 +16,7 @@ struct Options { uint16_t raw_file_merge_trigger_number = 100; size_t raw_to_index_trigger_size = 100000; std::shared_ptr pMetaOptions; + Env* env; }; // Options From b162ab0a04ec3576715a98bbedf8bbe48fb768ce Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 15 Apr 2019 15:45:33 +0800 Subject: [PATCH 29/34] fix(db): fix compile error Former-commit-id: d6d70bd6a6607001e90b920726c62af48b5341f5 --- cpp/src/db/memvectors.cpp | 42 ++++++++++++++++++++++----------------- cpp/src/db/memvectors.h | 9 ++++++--- cpp/src/db/options.cpp | 1 + cpp/src/db/options.h | 6 ++---- cpp/src/db/types.h | 2 +- 5 files changed, 34 insertions(+), 26 deletions(-) diff --git a/cpp/src/db/memvectors.cpp b/cpp/src/db/memvectors.cpp index f76589a628..abc6b67dfd 100644 --- a/cpp/src/db/memvectors.cpp +++ b/cpp/src/db/memvectors.cpp @@ -1,19 +1,21 @@ -#include -#include -#include +#include +#include +#include #include "memvectors.h" #include "db_meta.h" -namespace vecengine { +namespace zilliz { +namespace vecwise { +namespace engine { MemVectors::MemVectors(size_t dimension_, const std::string& file_location_) : - _file_location(file_location_), + _file_location(file_location_.c_str()), _pIdGenerator(new SimpleIDGenerator()), _dimension(dimension_), _pInnerIndex(new faiss::IndexFlat(_dimension)), - _pIdMapIndex = new faiss::IndexIDMap(_pInnerIndex) { + _pIdMapIndex(new faiss::IndexIDMap(_pInnerIndex)) { } void MemVectors::add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { @@ -52,14 +54,14 @@ MemVectors::~MemVectors() { * MemManager */ -VectorsPtr MemManager::get_mem_by_group(const std::string& group_id_) { - auto memIt = _memMap.find(group_id_); - if memIt != _memMap.end() { - return &(memIt->second); +VectorsPtr MemManager::get_mem_by_group(const std::string& group_id) { + auto memIt = _memMap.find(group_id); + if (memIt != _memMap.end()) { + return memIt->second; } GroupSchema group_info; - Status status = _pMeta->get_group(group_id_, group_info); + Status status = _pMeta->get_group(group_id, group_info); if (!status.ok()) { return nullptr; } @@ -76,15 +78,17 @@ Status MemManager::add_vectors(const std::string& group_id_, return add_vectors_no_lock(group_id_, n_, vectors_, vector_ids_); } -Status MemManager::add_vectors_no_lock(const std::string& group_id_, +Status MemManager::add_vectors_no_lock(const std::string& group_id, size_t n, const float* vectors, - IDNumbers& vector_ids_) { - auto mem = get_mem_by_group(group_id_); + IDNumbers& vector_ids) { + auto mem = get_mem_by_group(group_id); if (mem == nullptr) { - return Status::NotFound("Group " + group_id_ " not found!"); + return Status::NotFound("Group " + group_id + " not found!"); } - return mem->add(n, vectors, vector_ids_); + mem->add(n, vectors, vector_ids); + + return Status::OK(); } Status MemManager::mark_memory_as_immutable() { @@ -111,11 +115,13 @@ Status MemManager::mark_memory_as_immutable() { Status MemManager::serialize() { mark_memory_as_immutable(); for (auto& mem : _immMems) { - mem->serialize() + mem->serialize(); } _immMems.clear(); /* _last_compact_time = std::time(nullptr); */ } -} // namespace vecengine +} // namespace engine +} // namespace vecwise +} // namespace zilliz diff --git a/cpp/src/db/memvectors.h b/cpp/src/db/memvectors.h index 034d039010..f60f37af88 100644 --- a/cpp/src/db/memvectors.h +++ b/cpp/src/db/memvectors.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include "id_generators.h" #include "status.h" @@ -21,7 +23,7 @@ class MemVectors { public: explicit MemVectors(size_t dimension_, const std::string& file_location_); - IDNumbers&& add(size_t n, const float* vectors); + void add(size_t n_, const float* vectors_, IDNumbers& vector_ids_); size_t total() const; @@ -32,7 +34,7 @@ public: ~MemVectors(); private: - std::string _file_location; + const char* _file_location; IDGenerator* _pIdGenerator; size_t _dimension; faiss::Index* _pInnerIndex; @@ -42,10 +44,10 @@ private: class Meta; +typedef std::shared_ptr VectorsPtr; class MemManager { public: - typedef std::shared_ptr VectorsPtr; MemManager(const std::shared_ptr& meta_) : _pMeta(meta_), _last_compact_time(std::time(nullptr)) {} @@ -59,6 +61,7 @@ public: private: Status add_vectors_no_lock(const std::string& group_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_); + Status mark_memory_as_immutable(); typedef std::map MemMap; typedef std::vector ImmMemPool; diff --git a/cpp/src/db/options.cpp b/cpp/src/db/options.cpp index dd28cdb7dd..be572c232f 100644 --- a/cpp/src/db/options.cpp +++ b/cpp/src/db/options.cpp @@ -1,4 +1,5 @@ #include "options.h" +#include "env.h" namespace zilliz { namespace vecwise { diff --git a/cpp/src/db/options.h b/cpp/src/db/options.h index aefd094edf..807bae38b2 100644 --- a/cpp/src/db/options.h +++ b/cpp/src/db/options.h @@ -1,5 +1,4 @@ -#ifndef VECENGINE_OPTIONS_H_ -#define VECENGINE_OPTIONS_H_ +#pragma once #include #include @@ -12,6 +11,7 @@ class MetaOptions; class Env; struct Options { + Options(); uint16_t memory_sync_interval = 10; uint16_t raw_file_merge_trigger_number = 100; size_t raw_to_index_trigger_size = 100000; @@ -39,5 +39,3 @@ struct DBMetaOptions : public MetaOptions { } // namespace engine } // namespace vecwise } // namespace zilliz - -#endif // VECENGINE_OPTIONS_H_ diff --git a/cpp/src/db/types.h b/cpp/src/db/types.h index 7fd33a0787..c9ed274def 100644 --- a/cpp/src/db/types.h +++ b/cpp/src/db/types.h @@ -6,7 +6,7 @@ namespace zilliz { namespace vecwise { namespace engine { -typedef uint64_t IDNumber; +typedef long IDNumber; typedef IDNumber* IDNumberPtr; typedef std::vector IDNumbers; From bf0afceec32969961ad94e89ef155b788d4ca235 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 15 Apr 2019 16:34:47 +0800 Subject: [PATCH 30/34] fix(db): fix runtime error due to missing return value Former-commit-id: daa9d86441d94dd0c3d82a49d1d276676915f124 --- cpp/src/db/memvectors.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cpp/src/db/memvectors.cpp b/cpp/src/db/memvectors.cpp index abc6b67dfd..0dc019b733 100644 --- a/cpp/src/db/memvectors.cpp +++ b/cpp/src/db/memvectors.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include "memvectors.h" #include "db_meta.h" @@ -74,7 +75,7 @@ Status MemManager::add_vectors(const std::string& group_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_) { - std::lock_guard lock(_mutex); + std::unique_lock lock(_mutex); return add_vectors_no_lock(group_id_, n_, vectors_, vector_ids_); } @@ -92,11 +93,12 @@ Status MemManager::add_vectors_no_lock(const std::string& group_id, } Status MemManager::mark_memory_as_immutable() { - std::lock_guard lock(_mutex); + std::unique_lock lock(_mutex); for (auto& kv: _memMap) { _immMems.push_back(kv.second); } _memMap.clear(); + return Status::OK(); } /* bool MemManager::need_serialize(double interval) { */ @@ -118,7 +120,7 @@ Status MemManager::serialize() { mem->serialize(); } _immMems.clear(); - /* _last_compact_time = std::time(nullptr); */ + return Status::OK(); } From 7d76589dbda0ce2183942433be9edfac0ef9cecb Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 15 Apr 2019 20:44:11 +0800 Subject: [PATCH 31/34] fix(db): fix mem runtime error Former-commit-id: e173bfdd8760a56733122c03128e9d6137171418 --- cpp/src/db/memvectors.cpp | 20 ++++++++++++++++---- cpp/src/db/memvectors.h | 12 +++++++++--- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/cpp/src/db/memvectors.cpp b/cpp/src/db/memvectors.cpp index 0dc019b733..2bafe3a9de 100644 --- a/cpp/src/db/memvectors.cpp +++ b/cpp/src/db/memvectors.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include "memvectors.h" #include "db_meta.h" @@ -12,7 +14,7 @@ namespace vecwise { namespace engine { MemVectors::MemVectors(size_t dimension_, const std::string& file_location_) : - _file_location(file_location_.c_str()), + _file_location(file_location_), _pIdGenerator(new SimpleIDGenerator()), _dimension(dimension_), _pInnerIndex(new faiss::IndexFlat(_dimension)), @@ -20,8 +22,11 @@ MemVectors::MemVectors(size_t dimension_, const std::string& file_location_) : } void MemVectors::add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { - vector_ids_ = _pIdGenerator->getNextIDNumbers(n_); + _pIdGenerator->getNextIDNumbers(n_, vector_ids_); _pIdMapIndex->add_with_ids(n_, vectors_, &vector_ids_[0]); + for(auto i=0 ; igetNextIDNumber(); */ + /* faiss::write_index(_pIdMapIndex, ss.str().c_str()); */ + /* std::cout << _pIdMapIndex->ntotal << std::endl; */ + /* std::cout << _file_location << std::endl; */ + faiss::write_index(_pIdMapIndex, _file_location.c_str()); } MemVectors::~MemVectors() { @@ -66,6 +76,7 @@ VectorsPtr MemManager::get_mem_by_group(const std::string& group_id) { if (!status.ok()) { return nullptr; } + _memMap[group_id] = std::shared_ptr(new MemVectors(group_info.dimension, group_info.next_file_location)); return _memMap[group_id]; @@ -83,7 +94,7 @@ Status MemManager::add_vectors_no_lock(const std::string& group_id, size_t n, const float* vectors, IDNumbers& vector_ids) { - auto mem = get_mem_by_group(group_id); + std::shared_ptr mem = get_mem_by_group(group_id); if (mem == nullptr) { return Status::NotFound("Group " + group_id + " not found!"); } @@ -97,6 +108,7 @@ Status MemManager::mark_memory_as_immutable() { for (auto& kv: _memMap) { _immMems.push_back(kv.second); } + _memMap.clear(); return Status::OK(); } diff --git a/cpp/src/db/memvectors.h b/cpp/src/db/memvectors.h index f60f37af88..f9a0bba1f2 100644 --- a/cpp/src/db/memvectors.h +++ b/cpp/src/db/memvectors.h @@ -33,8 +33,14 @@ public: ~MemVectors(); + const std::string& location() const { return _file_location; } + private: - const char* _file_location; + MemVectors() = delete; + MemVectors(const MemVectors&) = delete; + MemVectors& operator=(const MemVectors&) = delete; + + const std::string _file_location; IDGenerator* _pIdGenerator; size_t _dimension; faiss::Index* _pInnerIndex; @@ -49,7 +55,7 @@ typedef std::shared_ptr VectorsPtr; class MemManager { public: MemManager(const std::shared_ptr& meta_) - : _pMeta(meta_), _last_compact_time(std::time(nullptr)) {} + : _pMeta(meta_) /*_last_compact_time(std::time(nullptr))*/ {} VectorsPtr get_mem_by_group(const std::string& group_id_); @@ -68,7 +74,7 @@ private: MemMap _memMap; ImmMemPool _immMems; std::shared_ptr _pMeta; - std::time_t _last_compact_time; + /* std::time_t _last_compact_time; */ std::mutex _mutex; }; // MemManager From 7296ca7084ed3f7b2f96ccc1de8a115be78d6f18 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 15 Apr 2019 20:45:21 +0800 Subject: [PATCH 32/34] fix(db): fix id generator runtime error Former-commit-id: 0de40fb0a90c1f1a09d7f800e523d811a2031caa --- cpp/src/db/id_generators.cpp | 8 ++++---- cpp/src/db/id_generators.h | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/src/db/id_generators.cpp b/cpp/src/db/id_generators.cpp index 3541048413..bb7da2c70d 100644 --- a/cpp/src/db/id_generators.cpp +++ b/cpp/src/db/id_generators.cpp @@ -1,5 +1,6 @@ #include #include +#include #include "id_generators.h" @@ -17,18 +18,17 @@ IDNumber SimpleIDGenerator::getNextIDNumber() { return micros * MAX_IDS_PER_MICRO; } -IDNumbers&& SimpleIDGenerator::getNextIDNumbers(size_t n) { +void SimpleIDGenerator::getNextIDNumbers(size_t n, IDNumbers& ids) { assert(n < MAX_IDS_PER_MICRO); auto now = std::chrono::system_clock::now(); auto micros = std::chrono::duration_cast( now.time_since_epoch()).count(); micros *= MAX_IDS_PER_MICRO; - IDNumbers ids = IDNumbers(n); + ids.clear(); for (int pos=0; pos Date: Mon, 15 Apr 2019 20:46:19 +0800 Subject: [PATCH 33/34] faker(db): mock some testing data Former-commit-id: 29c187ff28d41d70f2049fcaa931538f0acd0654 --- cpp/src/db/db_meta_impl.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cpp/src/db/db_meta_impl.cpp b/cpp/src/db/db_meta_impl.cpp index 23ca5dd6d2..7ab9f65000 100644 --- a/cpp/src/db/db_meta_impl.cpp +++ b/cpp/src/db/db_meta_impl.cpp @@ -1,4 +1,7 @@ +#include +#include #include "db_meta_impl.h" +#include "id_generators.h" namespace zilliz { namespace vecwise { @@ -23,6 +26,12 @@ Status DBMetaImpl::add_group(const GroupOptions& options_, Status DBMetaImpl::get_group(const std::string& group_id_, GroupSchema& group_info_) { //PXU TODO + std::stringstream ss; + SimpleIDGenerator g; + ss.str(""); + ss << "/tmp/test/" << g.getNextIDNumber() << ".log"; + group_info_.dimension = 64; + group_info_.next_file_location = ss.str(); return Status::OK(); } From dbed12bedff39c6f0510294bc14623aa03cba126 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Tue, 16 Apr 2019 09:48:58 +0800 Subject: [PATCH 34/34] feat(db): add search api for db Former-commit-id: 71e43c9b47c77b63968722d0595c97a474d33c9d --- cpp/src/db/db.h | 3 +++ cpp/src/db/db_impl.cpp | 6 ++++++ cpp/src/db/db_impl.h | 3 +++ cpp/src/db/types.h | 3 +++ 4 files changed, 15 insertions(+) diff --git a/cpp/src/db/db.h b/cpp/src/db/db.h index abaa498421..c14822b041 100644 --- a/cpp/src/db/db.h +++ b/cpp/src/db/db.h @@ -29,6 +29,9 @@ public: virtual Status add_vectors(const std::string& group_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) = 0; + virtual Status search(const std::string& group_id, size_t k, size_t nq, + const float* vectors, QueryResults& results) = 0; + DB() = default; DB(const DB&) = delete; DB& operator=(const DB&) = delete; diff --git a/cpp/src/db/db_impl.cpp b/cpp/src/db/db_impl.cpp index a23fbe1db6..87c83e7027 100644 --- a/cpp/src/db/db_impl.cpp +++ b/cpp/src/db/db_impl.cpp @@ -52,6 +52,12 @@ Status DBImpl::add_vectors(const std::string& group_id_, } } +Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, + const float* vectors, QueryResults& results) { + // PXU TODO + return Status::OK(); +} + void DBImpl::start_timer_task(int interval_) { std::thread bg_task(&DBImpl::background_timer_task, this, interval_); bg_task.detach(); diff --git a/cpp/src/db/db_impl.h b/cpp/src/db/db_impl.h index d9e3a3d7c9..b8e45f8006 100644 --- a/cpp/src/db/db_impl.h +++ b/cpp/src/db/db_impl.h @@ -31,6 +31,9 @@ public: virtual Status add_vectors(const std::string& group_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) override; + virtual Status search(const std::string& group_id, size_t k, size_t nq, + const float* vectors, QueryResults& results) override; + virtual ~DBImpl(); private: diff --git a/cpp/src/db/types.h b/cpp/src/db/types.h index c9ed274def..be538b9b1c 100644 --- a/cpp/src/db/types.h +++ b/cpp/src/db/types.h @@ -10,6 +10,9 @@ typedef long IDNumber; typedef IDNumber* IDNumberPtr; typedef std::vector IDNumbers; +typedef std::vector QueryResult; +typedef std::vector QueryResults; + } // namespace engine } // namespace vecwise