diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 6364698100..d80dfda958 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -1,39 +1,37 @@ +# All the following configurations are default values. + server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # port range: 1025 ~ 65534 - mode: single # deployment type: single, cluster, read_only + address: 0.0.0.0 # milvus server ip address (IPv4) + port: 19530 # port range: 1025 ~ 65534 + deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable time_zone: UTC+8 db_config: - path: @MILVUS_DB_PATH@ # milvus database path - slave_path: # secondary database path, split by semicolon + primary_path: @MILVUS_DB_PATH@ # path used to store data and meta + secondary_path: # path used to store data only, split by semicolon - # URI format: dialect://username:password@host:port/database - # All parts except dialect are optional, but you MUST include the delimiters - # Currently dialect supports mysql or sqlite - backend_url: sqlite://:@:/ + backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database + # Keep 'dialect://:@:/', and replace other texts with real values. + # Replace 'dialect' with 'mysql' or 'sqlite' - archive_disk_threshold: 0 # GB, file will be archived when disk usage exceed, 0 for no limit - archive_days_threshold: 0 # DAYS, older files will be archived, 0 for no limit - buffer_size: 4 # GB, maximum insert buffer size allowed - build_index_gpu: 0 # gpu id used for building index + insert_buffer_size: 4 # GB, maximum insert buffer size allowed + build_index_gpu: 0 # gpu id used for building index metric_config: - auto_bootup: off # whether enable monitoring when bootup - collector: prometheus # prometheus + enable_monitor: false # enable monitoring or not + collector: prometheus # prometheus prometheus_config: - port: 8080 # port prometheus used to fetch metrics + port: 8080 # port prometheus used to fetch metrics cache_config: - cpu_mem_capacity: 16 # GB, CPU memory size used for cache - cpu_mem_threshold: 0.85 # percent of data kept when cache cleanup triggered - cache_insert_data: false # whether load data into cache when insert + cpu_mem_capacity: 16 # GB, CPU memory used for cache + cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered + cache_insert_data: false # whether load inserted data into cache engine_config: blas_threshold: 20 resource_config: - mode: simple - pool: + resource_pool: - cpu - gpu0 diff --git a/cpp/src/cache/Cache.h b/cpp/src/cache/Cache.h index b2a57ebdfb..b526e9339f 100644 --- a/cpp/src/cache/Cache.h +++ b/cpp/src/cache/Cache.h @@ -32,30 +32,40 @@ namespace cache { template class Cache { -public: + public: //mem_capacity, units:GB Cache(int64_t capacity_gb, uint64_t cache_max_count); ~Cache() = default; - int64_t usage() const { return usage_; } - int64_t capacity() const { return capacity_; } //unit: BYTE + int64_t usage() const { + return usage_; + } + + int64_t capacity() const { + return capacity_; + } //unit: BYTE void set_capacity(int64_t capacity); //unit: BYTE - double freemem_percent() const { return freemem_percent_; }; - void set_freemem_percent(double percent) { freemem_percent_ = percent; } + double freemem_percent() const { + return freemem_percent_; + } + + void set_freemem_percent(double percent) { + freemem_percent_ = percent; + } size_t size() const; - bool exists(const std::string& key); - ItemObj get(const std::string& key); - void insert(const std::string& key, const ItemObj& item); - void erase(const std::string& key); + bool exists(const std::string &key); + ItemObj get(const std::string &key); + void insert(const std::string &key, const ItemObj &item); + void erase(const std::string &key); void print(); void clear(); -private: + private: void free_memory(); -private: + private: int64_t usage_; int64_t capacity_; double freemem_percent_; @@ -64,8 +74,8 @@ private: mutable std::mutex mutex_; }; -} // cache -} // milvus -} // zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz -#include "cache/Cache.inl" \ No newline at end of file +#include "cache/Cache.inl" diff --git a/cpp/src/cache/Cache.inl b/cpp/src/cache/Cache.inl index 3eaffc4556..ebd8f3c25d 100644 --- a/cpp/src/cache/Cache.inl +++ b/cpp/src/cache/Cache.inl @@ -33,29 +33,33 @@ Cache::Cache(int64_t capacity, uint64_t cache_max_count) } template -void Cache::set_capacity(int64_t capacity) { - if(capacity > 0) { +void +Cache::set_capacity(int64_t capacity) { + if (capacity > 0) { capacity_ = capacity; free_memory(); } } template -size_t Cache::size() const { +size_t +Cache::size() const { std::lock_guard lock(mutex_); return lru_.size(); } template -bool Cache::exists(const std::string& key) { +bool +Cache::exists(const std::string &key) { std::lock_guard lock(mutex_); return lru_.exists(key); } template -ItemObj Cache::get(const std::string& key) { +ItemObj +Cache::get(const std::string &key) { std::lock_guard lock(mutex_); - if(!lru_.exists(key)){ + if (!lru_.exists(key)) { return nullptr; } @@ -63,8 +67,9 @@ ItemObj Cache::get(const std::string& key) { } template -void Cache::insert(const std::string& key, const ItemObj& item) { - if(item == nullptr) { +void +Cache::insert(const std::string &key, const ItemObj &item) { + if (item == nullptr) { return; } @@ -80,7 +85,7 @@ void Cache::insert(const std::string& key, const ItemObj& item) { //if key already exist, subtract old item size if (lru_.exists(key)) { - const ItemObj& old_item = lru_.get(key); + const ItemObj &old_item = lru_.get(key); usage_ -= old_item->size(); } @@ -107,13 +112,14 @@ void Cache::insert(const std::string& key, const ItemObj& item) { } template -void Cache::erase(const std::string& key) { +void +Cache::erase(const std::string &key) { std::lock_guard lock(mutex_); - if(!lru_.exists(key)){ + if (!lru_.exists(key)) { return; } - const ItemObj& old_item = lru_.get(key); + const ItemObj &old_item = lru_.get(key); usage_ -= old_item->size(); SERVER_LOG_DEBUG << "Erase " << key << " size: " << old_item->size(); @@ -122,7 +128,8 @@ void Cache::erase(const std::string& key) { } template -void Cache::clear() { +void +Cache::clear() { std::lock_guard lock(mutex_); lru_.clear(); usage_ = 0; @@ -131,12 +138,13 @@ void Cache::clear() { /* free memory space when CACHE occupation exceed its capacity */ template -void Cache::free_memory() { +void +Cache::free_memory() { if (usage_ <= capacity_) return; int64_t threshhold = capacity_ * freemem_percent_; int64_t delta_size = usage_ - threshhold; - if(delta_size <= 0) { + if (delta_size <= 0) { delta_size = 1;//ensure at least one item erased } @@ -148,8 +156,8 @@ void Cache::free_memory() { auto it = lru_.rbegin(); while (it != lru_.rend() && released_size < delta_size) { - auto& key = it->first; - auto& obj_ptr = it->second; + auto &key = it->first; + auto &obj_ptr = it->second; key_array.emplace(key); released_size += obj_ptr->size(); @@ -159,7 +167,7 @@ void Cache::free_memory() { SERVER_LOG_DEBUG << "to be released memory size: " << released_size; - for (auto& key : key_array) { + for (auto &key : key_array) { erase(key); } @@ -167,7 +175,8 @@ void Cache::free_memory() { } template -void Cache::print() { +void +Cache::print() { size_t cache_count = 0; { std::lock_guard lock(mutex_); @@ -179,7 +188,7 @@ void Cache::print() { SERVER_LOG_DEBUG << "[Cache capacity]: " << capacity_ << " bytes"; } -} // cache -} // milvus -} // zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/CacheMgr.h b/cpp/src/cache/CacheMgr.h index 1dd630e14d..53004d10b2 100644 --- a/cpp/src/cache/CacheMgr.h +++ b/cpp/src/cache/CacheMgr.h @@ -22,22 +22,25 @@ #include "utils/Log.h" #include "metrics/Metrics.h" +#include +#include + namespace zilliz { namespace milvus { namespace cache { template class CacheMgr { -public: + public: virtual uint64_t ItemCount() const; - virtual bool ItemExists(const std::string& key); + virtual bool ItemExists(const std::string &key); - virtual ItemObj GetItem(const std::string& key); + virtual ItemObj GetItem(const std::string &key); - virtual void InsertItem(const std::string& key, const ItemObj& data); + virtual void InsertItem(const std::string &key, const ItemObj &data); - virtual void EraseItem(const std::string& key); + virtual void EraseItem(const std::string &key); virtual void PrintInfo(); @@ -47,18 +50,17 @@ public: int64_t CacheCapacity() const; void SetCapacity(int64_t capacity); -protected: + protected: CacheMgr(); virtual ~CacheMgr(); -protected: + protected: using CachePtr = std::shared_ptr>; CachePtr cache_; }; +} // namespace cache +} // namespace milvus +} // namespace zilliz -} -} -} - -#include "cache/CacheMgr.inl" \ No newline at end of file +#include "cache/CacheMgr.inl" diff --git a/cpp/src/cache/CacheMgr.inl b/cpp/src/cache/CacheMgr.inl index 36c3d94642..b0c47b3dc3 100644 --- a/cpp/src/cache/CacheMgr.inl +++ b/cpp/src/cache/CacheMgr.inl @@ -30,18 +30,20 @@ CacheMgr::~CacheMgr() { } template -uint64_t CacheMgr::ItemCount() const { - if(cache_ == nullptr) { +uint64_t +CacheMgr::ItemCount() const { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; } - return (uint64_t)(cache_->size()); + return (uint64_t) (cache_->size()); } template -bool CacheMgr::ItemExists(const std::string& key) { - if(cache_ == nullptr) { +bool +CacheMgr::ItemExists(const std::string &key) { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return false; } @@ -50,8 +52,9 @@ bool CacheMgr::ItemExists(const std::string& key) { } template -ItemObj CacheMgr::GetItem(const std::string& key) { - if(cache_ == nullptr) { +ItemObj +CacheMgr::GetItem(const std::string &key) { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return nullptr; } @@ -60,8 +63,9 @@ ItemObj CacheMgr::GetItem(const std::string& key) { } template -void CacheMgr::InsertItem(const std::string& key, const ItemObj& data) { - if(cache_ == nullptr) { +void +CacheMgr::InsertItem(const std::string &key, const ItemObj &data) { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -71,8 +75,9 @@ void CacheMgr::InsertItem(const std::string& key, const ItemObj& data) } template -void CacheMgr::EraseItem(const std::string& key) { - if(cache_ == nullptr) { +void +CacheMgr::EraseItem(const std::string &key) { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -82,8 +87,9 @@ void CacheMgr::EraseItem(const std::string& key) { } template -void CacheMgr::PrintInfo() { - if(cache_ == nullptr) { +void +CacheMgr::PrintInfo() { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -92,8 +98,9 @@ void CacheMgr::PrintInfo() { } template -void CacheMgr::ClearCache() { - if(cache_ == nullptr) { +void +CacheMgr::ClearCache() { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -102,8 +109,9 @@ void CacheMgr::ClearCache() { } template -int64_t CacheMgr::CacheUsage() const { - if(cache_ == nullptr) { +int64_t +CacheMgr::CacheUsage() const { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; } @@ -112,8 +120,9 @@ int64_t CacheMgr::CacheUsage() const { } template -int64_t CacheMgr::CacheCapacity() const { - if(cache_ == nullptr) { +int64_t +CacheMgr::CacheCapacity() const { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; } @@ -122,14 +131,15 @@ int64_t CacheMgr::CacheCapacity() const { } template -void CacheMgr::SetCapacity(int64_t capacity) { - if(cache_ == nullptr) { +void +CacheMgr::SetCapacity(int64_t capacity) { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; } cache_->set_capacity(capacity); } -} -} -} +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index 1f93ec92bd..d26004b2be 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -16,20 +16,22 @@ // under the License. -#include "CpuCacheMgr.h" +#include "cache/CpuCacheMgr.h" #include "server/Config.h" #include "utils/Log.h" +#include + namespace zilliz { namespace milvus { namespace cache { namespace { - constexpr int64_t unit = 1024 * 1024 * 1024; +constexpr int64_t unit = 1024 * 1024 * 1024; } CpuCacheMgr::CpuCacheMgr() { - server::Config& config = server::Config::GetInstance(); + server::Config &config = server::Config::GetInstance(); Status s; int32_t cpu_mem_cap; @@ -38,7 +40,7 @@ CpuCacheMgr::CpuCacheMgr() { SERVER_LOG_ERROR << s.message(); } int64_t cap = cpu_mem_cap * unit; - cache_ = std::make_shared>(cap, 1UL<<32); + cache_ = std::make_shared>(cap, 1UL << 32); float cpu_mem_threshold; s = config.GetCacheConfigCpuMemThreshold(cpu_mem_threshold); @@ -53,20 +55,22 @@ CpuCacheMgr::CpuCacheMgr() { } } -CpuCacheMgr* CpuCacheMgr::GetInstance() { +CpuCacheMgr * +CpuCacheMgr::GetInstance() { static CpuCacheMgr s_mgr; return &s_mgr; } -engine::VecIndexPtr CpuCacheMgr::GetIndex(const std::string& key) { +engine::VecIndexPtr +CpuCacheMgr::GetIndex(const std::string &key) { DataObjPtr obj = GetItem(key); - if(obj != nullptr) { + if (obj != nullptr) { return obj->data(); } return nullptr; } -} -} -} \ No newline at end of file +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/CpuCacheMgr.h b/cpp/src/cache/CpuCacheMgr.h index 5a2cdb599a..32a83c4cc0 100644 --- a/cpp/src/cache/CpuCacheMgr.h +++ b/cpp/src/cache/CpuCacheMgr.h @@ -20,21 +20,24 @@ #include "CacheMgr.h" #include "DataObj.h" +#include +#include + namespace zilliz { namespace milvus { namespace cache { class CpuCacheMgr : public CacheMgr { -private: + private: CpuCacheMgr(); -public: + public: //TODO: use smart pointer instead - static CpuCacheMgr* GetInstance(); + static CpuCacheMgr *GetInstance(); - engine::VecIndexPtr GetIndex(const std::string& key); + engine::VecIndexPtr GetIndex(const std::string &key); }; -} -} -} +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/DataObj.h b/cpp/src/cache/DataObj.h index f9215bc152..6ed3256eee 100644 --- a/cpp/src/cache/DataObj.h +++ b/cpp/src/cache/DataObj.h @@ -27,38 +27,43 @@ namespace milvus { namespace cache { class DataObj { -public: - DataObj(const engine::VecIndexPtr& index) - : index_(index) - {} + public: + explicit DataObj(const engine::VecIndexPtr &index) + : index_(index) { + } - DataObj(const engine::VecIndexPtr& index, int64_t size) - : index_(index), - size_(size) - {} + DataObj(const engine::VecIndexPtr &index, int64_t size) + : index_(index), + size_(size) { + } - engine::VecIndexPtr data() { return index_; } - const engine::VecIndexPtr& data() const { return index_; } + engine::VecIndexPtr data() { + return index_; + } + + const engine::VecIndexPtr &data() const { + return index_; + } int64_t size() const { - if(index_ == nullptr) { + if (index_ == nullptr) { return 0; } - if(size_ > 0) { + if (size_ > 0) { return size_; } return index_->Count() * index_->Dimension() * sizeof(float); } -private: + private: engine::VecIndexPtr index_ = nullptr; int64_t size_ = 0; }; using DataObjPtr = std::shared_ptr; -} -} -} \ No newline at end of file +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index 5c104ba0e7..ad68a6ebef 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -16,11 +16,13 @@ // under the License. -#include +#include "cache/GpuCacheMgr.h" #include "utils/Log.h" -#include "GpuCacheMgr.h" #include "server/Config.h" +#include +#include + namespace zilliz { namespace milvus { namespace cache { @@ -29,11 +31,11 @@ std::mutex GpuCacheMgr::mutex_; std::unordered_map GpuCacheMgr::instance_; namespace { - constexpr int64_t G_BYTE = 1024 * 1024 * 1024; +constexpr int64_t G_BYTE = 1024 * 1024 * 1024; } GpuCacheMgr::GpuCacheMgr() { - server::Config& config = server::Config::GetInstance(); + server::Config &config = server::Config::GetInstance(); Status s; int32_t gpu_mem_cap; @@ -42,7 +44,7 @@ GpuCacheMgr::GpuCacheMgr() { SERVER_LOG_ERROR << s.message(); } int32_t cap = gpu_mem_cap * G_BYTE; - cache_ = std::make_shared>(cap, 1UL<<32); + cache_ = std::make_shared>(cap, 1UL << 32); float gpu_mem_threshold; s = config.GetCacheConfigGpuMemThreshold(gpu_mem_threshold); @@ -57,7 +59,8 @@ GpuCacheMgr::GpuCacheMgr() { } } -GpuCacheMgr* GpuCacheMgr::GetInstance(uint64_t gpu_id) { +GpuCacheMgr * +GpuCacheMgr::GetInstance(uint64_t gpu_id) { if (instance_.find(gpu_id) == instance_.end()) { std::lock_guard lock(mutex_); if (instance_.find(gpu_id) == instance_.end()) { @@ -70,15 +73,16 @@ GpuCacheMgr* GpuCacheMgr::GetInstance(uint64_t gpu_id) { } } -engine::VecIndexPtr GpuCacheMgr::GetIndex(const std::string& key) { +engine::VecIndexPtr +GpuCacheMgr::GetIndex(const std::string &key) { DataObjPtr obj = GetItem(key); - if(obj != nullptr) { + if (obj != nullptr) { return obj->data(); } return nullptr; } -} -} -} \ No newline at end of file +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/GpuCacheMgr.h b/cpp/src/cache/GpuCacheMgr.h index 57539b6981..843a5ff67d 100644 --- a/cpp/src/cache/GpuCacheMgr.h +++ b/cpp/src/cache/GpuCacheMgr.h @@ -21,6 +21,7 @@ #include #include +#include namespace zilliz { namespace milvus { @@ -30,18 +31,18 @@ class GpuCacheMgr; using GpuCacheMgrPtr = std::shared_ptr; class GpuCacheMgr : public CacheMgr { -public: + public: GpuCacheMgr(); - static GpuCacheMgr* GetInstance(uint64_t gpu_id); + static GpuCacheMgr *GetInstance(uint64_t gpu_id); - engine::VecIndexPtr GetIndex(const std::string& key); + engine::VecIndexPtr GetIndex(const std::string &key); -private: + private: static std::mutex mutex_; static std::unordered_map instance_; }; -} -} -} +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/LRU.h b/cpp/src/cache/LRU.h index a4f4dcf5f2..5446dd0f14 100644 --- a/cpp/src/cache/LRU.h +++ b/cpp/src/cache/LRU.h @@ -22,6 +22,7 @@ #include #include #include +#include namespace zilliz { namespace milvus { @@ -29,14 +30,15 @@ namespace cache { template class LRU { -public: + public: typedef typename std::pair key_value_pair_t; typedef typename std::list::iterator list_iterator_t; typedef typename std::list::reverse_iterator reverse_list_iterator_t; - LRU(size_t max_size) : max_size_(max_size) {} + explicit LRU(size_t max_size) : max_size_(max_size) { + } - void put(const key_t& key, const value_t& value) { + void put(const key_t &key, const value_t &value) { auto it = cache_items_map_.find(key); cache_items_list_.push_front(key_value_pair_t(key, value)); if (it != cache_items_map_.end()) { @@ -53,7 +55,7 @@ public: } } - const value_t& get(const key_t& key) { + const value_t &get(const key_t &key) { auto it = cache_items_map_.find(key); if (it == cache_items_map_.end()) { throw std::range_error("There is no such key in cache"); @@ -63,7 +65,7 @@ public: } } - void erase(const key_t& key) { + void erase(const key_t &key) { auto it = cache_items_map_.find(key); if (it != cache_items_map_.end()) { cache_items_list_.erase(it->second); @@ -71,7 +73,7 @@ public: } } - bool exists(const key_t& key) const { + bool exists(const key_t &key) const { return cache_items_map_.find(key) != cache_items_map_.end(); } @@ -101,14 +103,14 @@ public: cache_items_map_.clear(); } -private: + private: std::list cache_items_list_; std::unordered_map cache_items_map_; size_t max_size_; list_iterator_t iter_; }; -} // cache -} // milvus -} // zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/ConfigMgr.cpp b/cpp/src/config/ConfigMgr.cpp index f39679fcdd..a7cfdc9fda 100644 --- a/cpp/src/config/ConfigMgr.cpp +++ b/cpp/src/config/ConfigMgr.cpp @@ -15,18 +15,19 @@ // specific language governing permissions and limitations // under the License. -#include "ConfigMgr.h" +#include "config/ConfigMgr.h" #include "YamlConfigMgr.h" namespace zilliz { namespace milvus { namespace server { -ConfigMgr * ConfigMgr::GetInstance() { +ConfigMgr * +ConfigMgr::GetInstance() { static YamlConfigMgr mgr; return &mgr; } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/ConfigMgr.h b/cpp/src/config/ConfigMgr.h index 3e546103b6..96a63aa2c8 100644 --- a/cpp/src/config/ConfigMgr.h +++ b/cpp/src/config/ConfigMgr.h @@ -20,6 +20,8 @@ #include "utils/Error.h" #include "ConfigNode.h" +#include + namespace zilliz { namespace milvus { namespace server { @@ -40,16 +42,16 @@ namespace server { class ConfigMgr { public: - static ConfigMgr* GetInstance(); + static ConfigMgr *GetInstance(); virtual ErrorCode LoadConfigFile(const std::string &filename) = 0; virtual void Print() const = 0;//will be deleted virtual std::string DumpString() const = 0; - virtual const ConfigNode& GetRootNode() const = 0; - virtual ConfigNode& GetRootNode() = 0; + virtual const ConfigNode &GetRootNode() const = 0; + virtual ConfigNode &GetRootNode() = 0; }; -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/ConfigNode.cpp b/cpp/src/config/ConfigNode.cpp index 88954c1bdf..87b080d572 100644 --- a/cpp/src/config/ConfigNode.cpp +++ b/cpp/src/config/ConfigNode.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "ConfigNode.h" +#include "config/ConfigNode.h" #include "utils/Error.h" #include "utils/Log.h" @@ -27,33 +27,34 @@ namespace zilliz { namespace milvus { namespace server { -void ConfigNode::Combine(const ConfigNode& target) { - const std::map& kv = target.GetConfig(); - for(auto itr = kv.begin(); itr != kv.end(); ++itr){ +void +ConfigNode::Combine(const ConfigNode &target) { + const std::map &kv = target.GetConfig(); + for (auto itr = kv.begin(); itr != kv.end(); ++itr) { config_[itr->first] = itr->second; } - const std::map >& sequences = target.GetSequences(); - for(auto itr = sequences.begin(); itr != sequences.end(); ++itr){ + const std::map > &sequences = target.GetSequences(); + for (auto itr = sequences.begin(); itr != sequences.end(); ++itr) { sequences_[itr->first] = itr->second; } - const std::map& children = target.GetChildren(); - for(auto itr = children.begin(); itr != children.end(); ++itr){ + const std::map &children = target.GetChildren(); + for (auto itr = children.begin(); itr != children.end(); ++itr) { children_[itr->first] = itr->second; } } //key/value pair config void -ConfigNode::SetValue(const std::string& key, const std::string& value) { +ConfigNode::SetValue(const std::string &key, const std::string &value) { config_[key] = value; } std::string -ConfigNode::GetValue(const std::string& param_key, const std::string& default_val) const { +ConfigNode::GetValue(const std::string ¶m_key, const std::string &default_val) const { auto ref = config_.find(param_key); - if(ref != config_.end()) { + if (ref != config_.end()) { return ref->second; } @@ -76,7 +77,7 @@ int32_t ConfigNode::GetInt32Value(const std::string ¶m_key, int32_t default_val) const { std::string val = GetValue(param_key); if (!val.empty()) { - return (int32_t)std::strtol(val.c_str(), nullptr, 10); + return (int32_t) std::strtol(val.c_str(), nullptr, 10); } else { return default_val; } @@ -112,25 +113,26 @@ ConfigNode::GetDoubleValue(const std::string ¶m_key, double default_val) con } } -const std::map& +const std::map & ConfigNode::GetConfig() const { return config_; -}; +} -void ConfigNode::ClearConfig() { +void +ConfigNode::ClearConfig() { config_.clear(); } //key/object config void -ConfigNode::AddChild(const std::string& type_name, const ConfigNode& config) { +ConfigNode::AddChild(const std::string &type_name, const ConfigNode &config) { children_[type_name] = config; } ConfigNode -ConfigNode::GetChild(const std::string& type_name) const { +ConfigNode::GetChild(const std::string &type_name) const { auto ref = children_.find(type_name); - if(ref != children_.end()) { + if (ref != children_.end()) { return ref->second; } @@ -138,25 +140,26 @@ ConfigNode::GetChild(const std::string& type_name) const { return nc; } -ConfigNode& +ConfigNode & ConfigNode::GetChild(const std::string &type_name) { return children_[type_name]; } void -ConfigNode::GetChildren(ConfigNodeArr& arr) const { +ConfigNode::GetChildren(ConfigNodeArr &arr) const { arr.clear(); - for(auto ref : children_){ + for (auto ref : children_) { arr.push_back(ref.second); } } -const std::map& +const std::map & ConfigNode::GetChildren() const { return children_; } -void ConfigNode::ClearChildren() { +void +ConfigNode::ClearChildren() { children_.clear(); } @@ -169,7 +172,7 @@ ConfigNode::AddSequenceItem(const std::string &key, const std::string &item) { std::vector ConfigNode::GetSequence(const std::string &key) const { auto itr = sequences_.find(key); - if(itr != sequences_.end()) { + if (itr != sequences_.end()) { return itr->second; } else { std::vector temp; @@ -177,29 +180,30 @@ ConfigNode::GetSequence(const std::string &key) const { } } -const std::map >& +const std::map > & ConfigNode::GetSequences() const { return sequences_; } -void ConfigNode::ClearSequences() { +void +ConfigNode::ClearSequences() { sequences_.clear(); } void -ConfigNode::PrintAll(const std::string& prefix) const { - for(auto& elem : config_) { +ConfigNode::PrintAll(const std::string &prefix) const { + for (auto &elem : config_) { SERVER_LOG_INFO << prefix << elem.first + ": " << elem.second; } - for(auto& elem : sequences_) { + for (auto &elem : sequences_) { SERVER_LOG_INFO << prefix << elem.first << ": "; - for(auto& str : elem.second) { + for (auto &str : elem.second) { SERVER_LOG_INFO << prefix << " - " << str; } } - for(auto& elem : children_) { + for (auto &elem : children_) { SERVER_LOG_INFO << prefix << elem.first << ": "; elem.second.PrintAll(prefix + " "); } @@ -209,18 +213,18 @@ std::string ConfigNode::DumpString(const std::string &prefix) const { std::stringstream str_buffer; const std::string endl = "\n"; - for(auto& elem : config_) { + for (auto &elem : config_) { str_buffer << prefix << elem.first << ": " << elem.second << endl; } - for(auto& elem : sequences_) { + for (auto &elem : sequences_) { str_buffer << prefix << elem.first << ": " << endl; - for(auto& str : elem.second) { + for (auto &str : elem.second) { str_buffer << prefix + " - " << str << endl; } } - for(auto& elem : children_) { + for (auto &elem : children_) { str_buffer << prefix << elem.first << ": " << endl; str_buffer << elem.second.DumpString(prefix + " ") << endl; } @@ -228,6 +232,6 @@ ConfigNode::DumpString(const std::string &prefix) const { return str_buffer.str(); } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/ConfigNode.h b/cpp/src/config/ConfigNode.h index 606e090d4b..d3fabc0655 100644 --- a/cpp/src/config/ConfigNode.h +++ b/cpp/src/config/ConfigNode.h @@ -30,7 +30,7 @@ typedef std::vector ConfigNodeArr; class ConfigNode { public: - void Combine(const ConfigNode& target); + void Combine(const ConfigNode &target); //key/value pair config void SetValue(const std::string &key, const std::string &value); @@ -42,23 +42,23 @@ class ConfigNode { float GetFloatValue(const std::string ¶m_key, float default_val = 0.0) const; double GetDoubleValue(const std::string ¶m_key, double default_val = 0.0) const; - const std::map& GetConfig() const; + const std::map &GetConfig() const; void ClearConfig(); //key/object config void AddChild(const std::string &type_name, const ConfigNode &config); ConfigNode GetChild(const std::string &type_name) const; - ConfigNode& GetChild(const std::string &type_name); + ConfigNode &GetChild(const std::string &type_name); void GetChildren(ConfigNodeArr &arr) const; - const std::map& GetChildren() const; + const std::map &GetChildren() const; void ClearChildren(); //key/sequence config void AddSequenceItem(const std::string &key, const std::string &item); std::vector GetSequence(const std::string &key) const; - const std::map >& GetSequences() const; + const std::map > &GetSequences() const; void ClearSequences(); void PrintAll(const std::string &prefix = "") const; @@ -70,6 +70,6 @@ class ConfigNode { std::map > sequences_; }; -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/YamlConfigMgr.cpp b/cpp/src/config/YamlConfigMgr.cpp index bddbc32245..1dd8a3d943 100644 --- a/cpp/src/config/YamlConfigMgr.cpp +++ b/cpp/src/config/YamlConfigMgr.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "YamlConfigMgr.h" +#include "config/YamlConfigMgr.h" #include "utils/Log.h" #include @@ -24,7 +24,8 @@ namespace zilliz { namespace milvus { namespace server { -ErrorCode YamlConfigMgr::LoadConfigFile(const std::string &filename) { +ErrorCode +YamlConfigMgr::LoadConfigFile(const std::string &filename) { struct stat directoryStat; int statOK = stat(filename.c_str(), &directoryStat); if (statOK != 0) { @@ -36,36 +37,40 @@ ErrorCode YamlConfigMgr::LoadConfigFile(const std::string &filename) { node_ = YAML::LoadFile(filename); LoadConfigNode(node_, config_); } - catch (YAML::Exception& e) { - SERVER_LOG_ERROR << "Failed to load config file: " << std::string(e.what ()); + catch (YAML::Exception &e) { + SERVER_LOG_ERROR << "Failed to load config file: " << std::string(e.what()); return SERVER_UNEXPECTED_ERROR; } return SERVER_SUCCESS; } -void YamlConfigMgr::Print() const { +void +YamlConfigMgr::Print() const { SERVER_LOG_INFO << "System config content:"; config_.PrintAll(); } -std::string YamlConfigMgr::DumpString() const { +std::string +YamlConfigMgr::DumpString() const { return config_.DumpString(""); } -const ConfigNode& YamlConfigMgr::GetRootNode() const { +const ConfigNode & +YamlConfigMgr::GetRootNode() const { return config_; } -ConfigNode& YamlConfigMgr::GetRootNode() { +ConfigNode & +YamlConfigMgr::GetRootNode() { return config_; } bool -YamlConfigMgr::SetConfigValue(const YAML::Node& node, - const std::string& key, - ConfigNode& config) { - if(node[key].IsDefined ()) { +YamlConfigMgr::SetConfigValue(const YAML::Node &node, + const std::string &key, + ConfigNode &config) { + if (node[key].IsDefined()) { config.SetValue(key, node[key].as()); return true; } @@ -73,10 +78,10 @@ YamlConfigMgr::SetConfigValue(const YAML::Node& node, } bool -YamlConfigMgr::SetChildConfig(const YAML::Node& node, - const std::string& child_name, - ConfigNode& config) { - if(node[child_name].IsDefined ()) { +YamlConfigMgr::SetChildConfig(const YAML::Node &node, + const std::string &child_name, + ConfigNode &config) { + if (node[child_name].IsDefined()) { ConfigNode sub_config; LoadConfigNode(node[child_name], sub_config); config.AddChild(child_name, sub_config); @@ -89,9 +94,9 @@ bool YamlConfigMgr::SetSequence(const YAML::Node &node, const std::string &child_name, ConfigNode &config) { - if(node[child_name].IsDefined ()) { + if (node[child_name].IsDefined()) { size_t cnt = node[child_name].size(); - for(size_t i = 0; i < cnt; i++){ + for (size_t i = 0; i < cnt; i++) { config.AddSequenceItem(child_name, node[child_name][i].as()); } return true; @@ -100,22 +105,22 @@ YamlConfigMgr::SetSequence(const YAML::Node &node, } void -YamlConfigMgr::LoadConfigNode(const YAML::Node& node, ConfigNode& config) { +YamlConfigMgr::LoadConfigNode(const YAML::Node &node, ConfigNode &config) { std::string key; for (YAML::const_iterator it = node.begin(); it != node.end(); ++it) { - if(!it->first.IsNull()){ + if (!it->first.IsNull()) { key = it->first.as(); } - if(node[key].IsScalar()) { + if (node[key].IsScalar()) { SetConfigValue(node, key, config); - } else if(node[key].IsMap()){ + } else if (node[key].IsMap()) { SetChildConfig(node, key, config); - } else if(node[key].IsSequence()){ + } else if (node[key].IsSequence()) { SetSequence(node, key, config); } } } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/YamlConfigMgr.h b/cpp/src/config/YamlConfigMgr.h index ef4d02c3ce..6b84943bf8 100644 --- a/cpp/src/config/YamlConfigMgr.h +++ b/cpp/src/config/YamlConfigMgr.h @@ -21,6 +21,7 @@ #include "ConfigNode.h" #include "utils/Error.h" +#include #include namespace zilliz { @@ -33,15 +34,15 @@ class YamlConfigMgr : public ConfigMgr { virtual void Print() const; virtual std::string DumpString() const; - virtual const ConfigNode& GetRootNode() const; - virtual ConfigNode& GetRootNode(); + virtual const ConfigNode &GetRootNode() const; + virtual ConfigNode &GetRootNode(); private: - bool SetConfigValue(const YAML::Node& node, - const std::string& key, - ConfigNode& config); + bool SetConfigValue(const YAML::Node &node, + const std::string &key, + ConfigNode &config); - bool SetChildConfig(const YAML::Node& node, + bool SetChildConfig(const YAML::Node &node, const std::string &name, ConfigNode &config); @@ -50,15 +51,13 @@ class YamlConfigMgr : public ConfigMgr { const std::string &child_name, ConfigNode &config); - void LoadConfigNode(const YAML::Node& node, ConfigNode& config); + void LoadConfigNode(const YAML::Node &node, ConfigNode &config); private: YAML::Node node_; ConfigNode config_; }; -} -} -} - - +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/Constants.h b/cpp/src/db/Constants.h index aa254b0bd6..2beb3c3a97 100644 --- a/cpp/src/db/Constants.h +++ b/cpp/src/db/Constants.h @@ -33,8 +33,8 @@ constexpr uint64_t MAX_TABLE_FILE_MEM = 128 * M; constexpr int VECTOR_TYPE_SIZE = sizeof(float); static constexpr uint64_t ONE_KB = K; -static constexpr uint64_t ONE_MB = ONE_KB*ONE_KB; -static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB; +static constexpr uint64_t ONE_MB = ONE_KB * ONE_KB; +static constexpr uint64_t ONE_GB = ONE_KB * ONE_MB; } // namespace engine } // namespace milvus diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 5fe6df491f..127812fc30 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -25,6 +25,7 @@ #include #include +#include namespace zilliz { namespace milvus { @@ -33,46 +34,45 @@ namespace engine { class Env; class DB { -public: + public: DB() = default; - DB(const DB&) = delete; - DB& operator=(const DB&) = delete; + DB(const DB &) = delete; + DB &operator=(const DB &) = delete; virtual ~DB() = default; virtual Status Start() = 0; virtual Status Stop() = 0; - virtual Status CreateTable(meta::TableSchema& table_schema_) = 0; - virtual Status DeleteTable(const std::string& table_id, const meta::DatesT& dates) = 0; - virtual Status DescribeTable(meta::TableSchema& table_schema_) = 0; - virtual Status HasTable(const std::string& table_id, bool& has_or_not_) = 0; - virtual Status AllTables(std::vector& table_schema_array) = 0; - virtual Status GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0; - virtual Status PreloadTable(const std::string& table_id) = 0; + virtual Status CreateTable(meta::TableSchema &table_schema_) = 0; + virtual Status DeleteTable(const std::string &table_id, const meta::DatesT &dates) = 0; + virtual Status DescribeTable(meta::TableSchema &table_schema_) = 0; + virtual Status HasTable(const std::string &table_id, bool &has_or_not_) = 0; + virtual Status AllTables(std::vector &table_schema_array) = 0; + virtual Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) = 0; + virtual Status PreloadTable(const std::string &table_id) = 0; virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0; - virtual Status InsertVectors(const std::string& table_id_, - uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0; + virtual Status InsertVectors(const std::string &table_id_, + uint64_t n, const float *vectors, IDNumbers &vector_ids_) = 0; - virtual Status Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float* vectors, QueryResults& results) = 0; + virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, + const float *vectors, QueryResults &results) = 0; - virtual Status Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0; + virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, + const float *vectors, const meta::DatesT &dates, QueryResults &results) = 0; - virtual Status Query(const std::string& table_id, const std::vector& file_ids, - uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, - const meta::DatesT& dates, QueryResults& results) = 0; + virtual Status Query(const std::string &table_id, const std::vector &file_ids, + uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors, + const meta::DatesT &dates, QueryResults &results) = 0; - virtual Status Size(uint64_t& result) = 0; + virtual Status Size(uint64_t &result) = 0; - virtual Status CreateIndex(const std::string& table_id, const TableIndex& index) = 0; - virtual Status DescribeIndex(const std::string& table_id, TableIndex& index) = 0; - virtual Status DropIndex(const std::string& table_id) = 0; + virtual Status CreateIndex(const std::string &table_id, const TableIndex &index) = 0; + virtual Status DescribeIndex(const std::string &table_id, TableIndex &index) = 0; + virtual Status DropIndex(const std::string &table_id) = 0; virtual Status DropAll() = 0; - }; // DB using DBPtr = std::shared_ptr; diff --git a/cpp/src/db/DBFactory.cpp b/cpp/src/db/DBFactory.cpp index 2eec49fc7a..edf70cccf8 100644 --- a/cpp/src/db/DBFactory.cpp +++ b/cpp/src/db/DBFactory.cpp @@ -16,7 +16,7 @@ // under the License. -#include "DBFactory.h" +#include "db/DBFactory.h" #include "DBImpl.h" #include "utils/Exception.h" #include "meta/MetaFactory.h" @@ -33,14 +33,16 @@ namespace zilliz { namespace milvus { namespace engine { -DBOptions DBFactory::BuildOption() { +DBOptions +DBFactory::BuildOption() { auto meta = MetaFactory::BuildOption(); - DBOptions options; + DBOptions options; options.meta_ = meta; return options; } -DBPtr DBFactory::Build(const DBOptions& options) { +DBPtr +DBFactory::Build(const DBOptions &options) { return std::make_shared(options); } diff --git a/cpp/src/db/DBFactory.h b/cpp/src/db/DBFactory.h index bb363118ed..1223a11c29 100644 --- a/cpp/src/db/DBFactory.h +++ b/cpp/src/db/DBFactory.h @@ -28,13 +28,12 @@ namespace milvus { namespace engine { class DBFactory { -public: + public: static DBOptions BuildOption(); - static DBPtr Build(const DBOptions& options); + static DBPtr Build(const DBOptions &options); }; - -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index a4945ca02e..ab3f3104af 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "DBImpl.h" +#include "db/DBImpl.h" #include "cache/CpuCacheMgr.h" #include "cache/GpuCacheMgr.h" #include "engine/EngineFactory.h" @@ -36,6 +36,7 @@ #include #include #include +#include #include namespace zilliz { @@ -48,10 +49,9 @@ constexpr uint64_t METRIC_ACTION_INTERVAL = 1; constexpr uint64_t COMPACT_ACTION_INTERVAL = 1; constexpr uint64_t INDEX_ACTION_INTERVAL = 1; -} +} // namespace - -DBImpl::DBImpl(const DBOptions& options) +DBImpl::DBImpl(const DBOptions &options) : options_(options), shutting_down_(true), compact_thread_pool_(1, 1), @@ -68,8 +68,9 @@ DBImpl::~DBImpl() { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //external api /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -Status DBImpl::Start() { - if (!shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::Start() { + if (!shutting_down_.load(std::memory_order_acquire)) { return Status::OK(); } @@ -77,7 +78,7 @@ Status DBImpl::Start() { shutting_down_.store(false, std::memory_order_release); //for distribute version, some nodes are read only - if (options_.mode_ != DBOptions::MODE::READ_ONLY) { + if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { ENGINE_LOG_TRACE << "StartTimerTasks"; bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this); } @@ -85,8 +86,9 @@ Status DBImpl::Start() { return Status::OK(); } -Status DBImpl::Stop() { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::Stop() { + if (shutting_down_.load(std::memory_order_acquire)) { return Status::OK(); } @@ -98,7 +100,7 @@ Status DBImpl::Stop() { //wait compaction/buildindex finish bg_timer_thread_.join(); - if (options_.mode_ != DBOptions::MODE::READ_ONLY) { + if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { meta_ptr_->CleanUp(); } @@ -106,12 +108,14 @@ Status DBImpl::Stop() { return Status::OK(); } -Status DBImpl::DropAll() { +Status +DBImpl::DropAll() { return meta_ptr_->DropAll(); } -Status DBImpl::CreateTable(meta::TableSchema& table_schema) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::CreateTable(meta::TableSchema &table_schema) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -120,8 +124,9 @@ Status DBImpl::CreateTable(meta::TableSchema& table_schema) { return meta_ptr_->CreateTable(temp_schema); } -Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::DeleteTable(const std::string &table_id, const meta::DatesT &dates) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -144,8 +149,9 @@ Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& date return Status::OK(); } -Status DBImpl::DescribeTable(meta::TableSchema& table_schema) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::DescribeTable(meta::TableSchema &table_schema) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -154,24 +160,27 @@ Status DBImpl::DescribeTable(meta::TableSchema& table_schema) { return stat; } -Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::HasTable(const std::string &table_id, bool &has_or_not) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } return meta_ptr_->HasTable(table_id, has_or_not); } -Status DBImpl::AllTables(std::vector& table_schema_array) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::AllTables(std::vector &table_schema_array) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } return meta_ptr_->AllTables(table_schema_array); } -Status DBImpl::PreloadTable(const std::string &table_id) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::PreloadTable(const std::string &table_id) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -189,10 +198,14 @@ Status DBImpl::PreloadTable(const std::string &table_id) { int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); int64_t available_size = cache_total - cache_usage; - for(auto &day_files : files) { + for (auto &day_files : files) { for (auto &file : day_files.second) { - ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, (MetricType)file.metric_type_, file.nlist_); - if(engine == nullptr) { + ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, + file.location_, + (EngineType) file.engine_type_, + (MetricType) file.metric_type_, + file.nlist_); + if (engine == nullptr) { ENGINE_LOG_ERROR << "Invalid engine type"; return Status(DB_ERROR, "Invalid engine type"); } @@ -215,33 +228,37 @@ Status DBImpl::PreloadTable(const std::string &table_id) { return Status::OK(); } -Status DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } return meta_ptr_->UpdateTableFlag(table_id, flag); } -Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::GetTableRowCount(const std::string &table_id, uint64_t &row_count) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } return meta_ptr_->Count(table_id, row_count); } -Status DBImpl::InsertVectors(const std::string& table_id_, - uint64_t n, const float* vectors, IDNumbers& vector_ids_) { +Status +DBImpl::InsertVectors(const std::string &table_id_, + uint64_t n, const float *vectors, IDNumbers &vector_ids_) { // ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache"; - if (shutting_down_.load(std::memory_order_acquire)){ + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } Status status; zilliz::milvus::server::CollectInsertMetrics metrics(n, status); status = mem_mgr_->InsertVectors(table_id_, n, vectors, vector_ids_); -// std::chrono::microseconds time_span = std::chrono::duration_cast(end_time - start_time); +// std::chrono::microseconds time_span = +// std::chrono::duration_cast(end_time - start_time); // double average_time = double(time_span.count()) / n; // ENGINE_LOG_DEBUG << "Insert vectors to cache finished"; @@ -249,14 +266,15 @@ Status DBImpl::InsertVectors(const std::string& table_id_, return status; } -Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { +Status +DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) { { std::unique_lock lock(build_index_mutex_); //step 1: check index difference TableIndex old_index; auto status = DescribeIndex(table_id, old_index); - if(!status.ok()) { + if (!status.ok()) { ENGINE_LOG_ERROR << "Failed to get table index info for table: " << table_id; return status; } @@ -264,7 +282,7 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) //step 2: update index info TableIndex new_index = index; new_index.metric_type_ = old_index.metric_type_;//dont change metric type, it was defined by CreateTable - if(!utils::IsSameIndex(old_index, new_index)) { + if (!utils::IsSameIndex(old_index, new_index)) { DropIndex(table_id); status = meta_ptr_->UpdateTableIndex(table_id, new_index); @@ -283,18 +301,18 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) //for IDMAP type, only wait all NEW file converted to RAW file //for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files std::vector file_types; - if(index.engine_type_ == (int)EngineType::FAISS_IDMAP) { + if (index.engine_type_ == (int) EngineType::FAISS_IDMAP) { file_types = { - (int) meta::TableFileSchema::NEW, - (int) meta::TableFileSchema::NEW_MERGE, + (int) meta::TableFileSchema::NEW, + (int) meta::TableFileSchema::NEW_MERGE, }; } else { file_types = { - (int) meta::TableFileSchema::RAW, - (int) meta::TableFileSchema::NEW, - (int) meta::TableFileSchema::NEW_MERGE, - (int) meta::TableFileSchema::NEW_INDEX, - (int) meta::TableFileSchema::TO_INDEX, + (int) meta::TableFileSchema::RAW, + (int) meta::TableFileSchema::NEW, + (int) meta::TableFileSchema::NEW_MERGE, + (int) meta::TableFileSchema::NEW_INDEX, + (int) meta::TableFileSchema::TO_INDEX, }; } @@ -304,11 +322,11 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) while (!file_ids.empty()) { ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times; - if(index.engine_type_ != (int)EngineType::FAISS_IDMAP) { + if (index.engine_type_ != (int) EngineType::FAISS_IDMAP) { status = meta_ptr_->UpdateTableFilesToIndex(table_id); } - std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100))); + std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10 * 1000, times * 100))); status = meta_ptr_->FilesByType(table_id, file_types, file_ids); times++; } @@ -316,18 +334,21 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) return Status::OK(); } -Status DBImpl::DescribeIndex(const std::string& table_id, TableIndex& index) { +Status +DBImpl::DescribeIndex(const std::string &table_id, TableIndex &index) { return meta_ptr_->DescribeTableIndex(table_id, index); } -Status DBImpl::DropIndex(const std::string& table_id) { +Status +DBImpl::DropIndex(const std::string &table_id) { ENGINE_LOG_DEBUG << "Drop index for table: " << table_id; return meta_ptr_->DropTableIndex(table_id); } -Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float *vectors, QueryResults &results) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, + const float *vectors, QueryResults &results) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -337,9 +358,10 @@ Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint6 return result; } -Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float* vectors, const meta::DatesT& dates, QueryResults& results) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, + const float *vectors, const meta::DatesT &dates, QueryResults &results) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -364,10 +386,11 @@ Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint6 return status; } -Status DBImpl::Query(const std::string& table_id, const std::vector& file_ids, - uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, - const meta::DatesT& dates, QueryResults& results) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::Query(const std::string &table_id, const std::vector &file_ids, + uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors, + const meta::DatesT &dates, QueryResults &results) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -395,7 +418,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector } } - if(file_id_array.empty()) { + if (file_id_array.empty()) { return Status(DB_ERROR, "Invalid file id"); } @@ -405,36 +428,37 @@ Status DBImpl::Query(const std::string& table_id, const std::vector return status; } -Status DBImpl::Size(uint64_t& result) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::Size(uint64_t &result) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } - return meta_ptr_->Size(result); + return meta_ptr_->Size(result); } - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //internal methods /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, - uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, - const meta::DatesT& dates, QueryResults& results) { - using namespace scheduler; +Status +DBImpl::QueryAsync(const std::string &table_id, const meta::TableFilesSchema &files, + uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors, + const meta::DatesT &dates, QueryResults &results) { server::CollectQueryMetrics metrics(nq); TimeRecorder rc(""); //step 1: get files to search - ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: " << dates.size(); - SearchJobPtr job = std::make_shared(0, k, nq, nprobe, vectors); - for (auto &file : files) { - TableFileSchemaPtr file_ptr = std::make_shared(file); + ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: " + << dates.size(); + scheduler::SearchJobPtr job = std::make_shared(0, k, nq, nprobe, vectors); + for (auto &file : files) { + scheduler::TableFileSchemaPtr file_ptr = std::make_shared(file); job->AddIndexFile(file_ptr); } //step 2: put search task to scheduler - JobMgrInst::GetInstance()->Put(job); + scheduler::JobMgrInst::GetInstance()->Put(job); job->WaitResult(); if (!job->GetStatus().ok()) { return job->GetStatus(); @@ -453,9 +477,12 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch // double search_percent = search_cost/total_cost; // double reduce_percent = reduce_cost/total_cost; // -// ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info << " percent: " << load_percent*100 << "%"; -// ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info << " percent: " << search_percent*100 << "%"; -// ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info << " percent: " << reduce_percent*100 << "%"; +// ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info +// << " percent: " << load_percent*100 << "%"; +// ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info +// << " percent: " << search_percent*100 << "%"; +// ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info +// << " percent: " << reduce_percent*100 << "%"; // } else { // ENGINE_LOG_DEBUG << "Engine load cost: " << load_info // << " search cost: " << search_info @@ -469,11 +496,12 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch return Status::OK(); } -void DBImpl::BackgroundTimerTask() { +void +DBImpl::BackgroundTimerTask() { Status status; server::SystemInfo::GetInstance().Init(); while (true) { - if (shutting_down_.load(std::memory_order_acquire)){ + if (shutting_down_.load(std::memory_order_acquire)) { WaitMergeFileFinish(); WaitBuildIndexFinish(); @@ -489,24 +517,27 @@ void DBImpl::BackgroundTimerTask() { } } -void DBImpl::WaitMergeFileFinish() { +void +DBImpl::WaitMergeFileFinish() { std::lock_guard lck(compact_result_mutex_); - for(auto& iter : compact_thread_results_) { + for (auto &iter : compact_thread_results_) { iter.wait(); } } -void DBImpl::WaitBuildIndexFinish() { +void +DBImpl::WaitBuildIndexFinish() { std::lock_guard lck(index_result_mutex_); - for(auto& iter : index_thread_results_) { + for (auto &iter : index_thread_results_) { iter.wait(); } } -void DBImpl::StartMetricTask() { +void +DBImpl::StartMetricTask() { static uint64_t metric_clock_tick = 0; metric_clock_tick++; - if(metric_clock_tick%METRIC_ACTION_INTERVAL != 0) { + if (metric_clock_tick % METRIC_ACTION_INTERVAL != 0) { return; } @@ -515,7 +546,7 @@ void DBImpl::StartMetricTask() { server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL); int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); int64_t cache_total = cache::CpuCacheMgr::GetInstance()->CacheCapacity(); - server::Metrics::GetInstance().CpuCacheUsageGaugeSet(cache_usage*100/cache_total); + server::Metrics::GetInstance().CpuCacheUsageGaugeSet(cache_usage * 100 / cache_total); server::Metrics::GetInstance().GpuCacheUsageGaugeSet(); uint64_t size; Size(size); @@ -533,25 +564,27 @@ void DBImpl::StartMetricTask() { ENGINE_LOG_TRACE << "Metric task finished"; } -Status DBImpl::MemSerialize() { +Status +DBImpl::MemSerialize() { std::lock_guard lck(mem_serialize_mutex_); std::set temp_table_ids; mem_mgr_->Serialize(temp_table_ids); - for(auto& id : temp_table_ids) { + for (auto &id : temp_table_ids) { compact_table_ids_.insert(id); } - if(!temp_table_ids.empty()) { + if (!temp_table_ids.empty()) { SERVER_LOG_DEBUG << "Insert cache serialized"; } return Status::OK(); } -void DBImpl::StartCompactionTask() { +void +DBImpl::StartCompactionTask() { static uint64_t compact_clock_tick = 0; compact_clock_tick++; - if(compact_clock_tick%COMPACT_ACTION_INTERVAL != 0) { + if (compact_clock_tick % COMPACT_ACTION_INTERVAL != 0) { return; } @@ -580,8 +613,9 @@ void DBImpl::StartCompactionTask() { } } -Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, - const meta::TableFilesSchema& files) { +Status +DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date, + const meta::TableFilesSchema &files) { ENGINE_LOG_DEBUG << "Merge files for table: " << table_id; //step 1: create table file @@ -598,13 +632,13 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, //step 2: merge files ExecutionEnginePtr index = - EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_, - (MetricType)table_file.metric_type_, table_file.nlist_); + EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType) table_file.engine_type_, + (MetricType) table_file.metric_type_, table_file.nlist_); meta::TableFilesSchema updated; - long index_size = 0; + int64_t index_size = 0; - for (auto& file : files) { + for (auto &file : files) { server::CollectMergeFilesMetrics metrics; index->Merge(file.location_); @@ -620,7 +654,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, //step 3: serialize to disk try { index->Serialize(); - } catch (std::exception& ex) { + } catch (std::exception &ex) { //typical error: out of disk space or permition denied std::string msg = "Serialize merged index encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; @@ -638,7 +672,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, //step 4: update table files state //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size //else set file type to RAW, no need to build index - if (table_file.engine_type_ != (int)EngineType::FAISS_IDMAP) { + if (table_file.engine_type_ != (int) EngineType::FAISS_IDMAP) { table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ? meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; } else { @@ -649,16 +683,17 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, updated.push_back(table_file); status = meta_ptr_->UpdateTableFiles(updated); ENGINE_LOG_DEBUG << "New merged file " << table_file.file_id_ << - " of size " << index->PhysicalSize() << " bytes"; + " of size " << index->PhysicalSize() << " bytes"; - if(options_.insert_cache_immediately_) { + if (options_.insert_cache_immediately_) { index->Cache(); } return status; } -Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { +Status +DBImpl::BackgroundMergeFiles(const std::string &table_id) { meta::DatePartionedTableFilesSchema raw_files; auto status = meta_ptr_->FilesToMerge(table_id, raw_files); if (!status.ok()) { @@ -667,7 +702,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { } bool has_merge = false; - for (auto& kv : raw_files) { + for (auto &kv : raw_files) { auto files = kv.second; if (files.size() < options_.merge_trigger_number_) { ENGINE_LOG_DEBUG << "Files number not greater equal than merge trigger number, skip merge action"; @@ -676,7 +711,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { has_merge = true; MergeFiles(table_id, kv.first, kv.second); - if (shutting_down_.load(std::memory_order_acquire)){ + if (shutting_down_.load(std::memory_order_acquire)) { ENGINE_LOG_DEBUG << "Server will shutdown, skip merge action for table: " << table_id; break; } @@ -685,17 +720,18 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { return Status::OK(); } -void DBImpl::BackgroundCompaction(std::set table_ids) { +void +DBImpl::BackgroundCompaction(std::set table_ids) { ENGINE_LOG_TRACE << " Background compaction thread start"; Status status; - for (auto& table_id : table_ids) { + for (auto &table_id : table_ids) { status = BackgroundMergeFiles(table_id); if (!status.ok()) { ENGINE_LOG_ERROR << "Merge files for table " << table_id << " failed: " << status.ToString(); } - if (shutting_down_.load(std::memory_order_acquire)){ + if (shutting_down_.load(std::memory_order_acquire)) { ENGINE_LOG_DEBUG << "Server will shutdown, skip merge action"; break; } @@ -703,8 +739,8 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); - int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes - if (options_.mode_ == DBOptions::MODE::CLUSTER) { + int ttl = 5 * meta::M_SEC;//default: file will be deleted after 5 minutes + if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { ttl = meta::D_SEC; } meta_ptr_->CleanUpFilesWithTTL(ttl); @@ -712,10 +748,11 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { ENGINE_LOG_TRACE << " Background compaction thread exit"; } -void DBImpl::StartBuildIndexTask(bool force) { +void +DBImpl::StartBuildIndexTask(bool force) { static uint64_t index_clock_tick = 0; index_clock_tick++; - if(!force && (index_clock_tick%INDEX_ACTION_INTERVAL != 0)) { + if (!force && (index_clock_tick % INDEX_ACTION_INTERVAL != 0)) { return; } @@ -740,11 +777,12 @@ void DBImpl::StartBuildIndexTask(bool force) { } } -Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { +Status +DBImpl::BuildIndex(const meta::TableFileSchema &file) { ExecutionEnginePtr to_index = - EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, - (MetricType)file.metric_type_, file.nlist_); - if(to_index == nullptr) { + EngineFactory::Build(file.dimension_, file.location_, (EngineType) file.engine_type_, + (MetricType) file.metric_type_, file.nlist_); + if (to_index == nullptr) { ENGINE_LOG_ERROR << "Invalid engine type"; return Status(DB_ERROR, "Invalid engine type"); } @@ -761,7 +799,8 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { meta::TableFileSchema table_file; table_file.table_id_ = file.table_id_; table_file.date_ = file.date_; - table_file.file_type_ = meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path + table_file.file_type_ = + meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path status = meta_ptr_->CreateTableFile(table_file); if (!status.ok()) { ENGINE_LOG_ERROR << "Failed to create table file: " << status.ToString(); @@ -773,16 +812,16 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { try { server::CollectBuildIndexMetrics metrics; - index = to_index->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_); + index = to_index->BuildIndex(table_file.location_, (EngineType) table_file.engine_type_); if (index == nullptr) { table_file.file_type_ = meta::TableFileSchema::TO_DELETE; status = meta_ptr_->UpdateTableFile(table_file); - ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; + ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ + << " to to_delete"; return status; } - - } catch (std::exception& ex) { + } catch (std::exception &ex) { //typical error: out of gpu memory std::string msg = "BuildIndex encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; @@ -791,7 +830,8 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { status = meta_ptr_->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; - std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" << std::endl; + std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" + << std::endl; return Status(DB_ERROR, msg); } @@ -799,7 +839,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { //step 4: if table has been deleted, dont save index file bool has_table = false; meta_ptr_->HasTable(file.table_id_, has_table); - if(!has_table) { + if (!has_table) { meta_ptr_->DeleteTableFiles(file.table_id_); return Status::OK(); } @@ -807,7 +847,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { //step 5: save index file try { index->Serialize(); - } catch (std::exception& ex) { + } catch (std::exception &ex) { //typical error: out of disk space or permition denied std::string msg = "Serialize index encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; @@ -817,7 +857,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; std::cout << "ERROR: failed to persist index file: " << table_file.location_ - << ", possible out of disk space" << std::endl; + << ", possible out of disk space" << std::endl; return Status(DB_ERROR, msg); } @@ -832,12 +872,12 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { meta::TableFilesSchema update_files = {table_file, origin_file}; status = meta_ptr_->UpdateTableFiles(update_files); - if(status.ok()) { + if (status.ok()) { ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size " << index->PhysicalSize() << " bytes" << " from file " << origin_file.file_id_; - if(options_.insert_cache_immediately_) { + if (options_.insert_cache_immediately_) { index->Cache(); } } else { @@ -850,8 +890,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { status = meta_ptr_->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; } - - } catch (std::exception& ex) { + } catch (std::exception &ex) { std::string msg = "Build index encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; return Status(DB_ERROR, msg); @@ -860,20 +899,21 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { return Status::OK(); } -void DBImpl::BackgroundBuildIndex() { +void +DBImpl::BackgroundBuildIndex() { ENGINE_LOG_TRACE << "Background build index thread start"; std::unique_lock lock(build_index_mutex_); meta::TableFilesSchema to_index_files; meta_ptr_->FilesToIndex(to_index_files); Status status; - for (auto& file : to_index_files) { + for (auto &file : to_index_files) { status = BuildIndex(file); if (!status.ok()) { ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString(); } - if (shutting_down_.load(std::memory_order_acquire)){ + if (shutting_down_.load(std::memory_order_acquire)) { ENGINE_LOG_DEBUG << "Server will shutdown, skip build index action"; break; } diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index bc9e453e5c..344e290445 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -29,7 +29,8 @@ #include #include #include - +#include +#include namespace zilliz { namespace milvus { @@ -68,11 +69,11 @@ class DBImpl : public DB { Status InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override; - Status CreateIndex(const std::string& table_id, const TableIndex& index) override; + Status CreateIndex(const std::string &table_id, const TableIndex &index) override; - Status DescribeIndex(const std::string& table_id, TableIndex& index) override; + Status DescribeIndex(const std::string &table_id, TableIndex &index) override; - Status DropIndex(const std::string& table_id) override; + Status DropIndex(const std::string &table_id) override; Status Query(const std::string &table_id, uint64_t k, @@ -123,7 +124,7 @@ class DBImpl : public DB { Status BackgroundMergeFiles(const std::string &table_id); void BackgroundCompaction(std::set table_ids); - void StartBuildIndexTask(bool force=false); + void StartBuildIndexTask(bool force = false); void BackgroundBuildIndex(); Status BuildIndex(const meta::TableFileSchema &); @@ -151,7 +152,6 @@ class DBImpl : public DB { std::list> index_thread_results_; std::mutex build_index_mutex_; - }; // DBImpl diff --git a/cpp/src/db/IDGenerator.cpp b/cpp/src/db/IDGenerator.cpp index a82ccee629..78c88b2c03 100644 --- a/cpp/src/db/IDGenerator.cpp +++ b/cpp/src/db/IDGenerator.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "IDGenerator.h" +#include "db/IDGenerator.h" #include #include @@ -29,16 +29,18 @@ IDGenerator::~IDGenerator() = default; constexpr size_t SimpleIDGenerator::MAX_IDS_PER_MICRO; -IDNumber SimpleIDGenerator::GetNextIDNumber() { +IDNumber +SimpleIDGenerator::GetNextIDNumber() { auto now = std::chrono::system_clock::now(); auto micros = std::chrono::duration_cast( - now.time_since_epoch()).count(); + now.time_since_epoch()).count(); return micros * MAX_IDS_PER_MICRO; } -void SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) { +void +SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) { if (n > MAX_IDS_PER_MICRO) { - NextIDNumbers(n-MAX_IDS_PER_MICRO, ids); + NextIDNumbers(n - MAX_IDS_PER_MICRO, ids); NextIDNumbers(MAX_IDS_PER_MICRO, ids); return; } @@ -48,20 +50,20 @@ void SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) { auto now = std::chrono::system_clock::now(); auto micros = std::chrono::duration_cast( - now.time_since_epoch()).count(); + now.time_since_epoch()).count(); micros *= MAX_IDS_PER_MICRO; - for (int pos=0; pos #include - namespace zilliz { namespace milvus { namespace engine { @@ -55,7 +54,6 @@ class SimpleIDGenerator : public IDGenerator { NextIDNumbers(size_t n, IDNumbers &ids); static constexpr size_t MAX_IDS_PER_MICRO = 1000; - }; // SimpleIDGenerator diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index fd52b851e2..4c82f3f736 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "Options.h" +#include "db/Options.h" #include "utils/Exception.h" #include "utils/Log.h" @@ -27,18 +27,20 @@ namespace zilliz { namespace milvus { namespace engine { -ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) { +ArchiveConf::ArchiveConf(const std::string &type, const std::string &criterias) { ParseType(type); ParseCritirias(criterias); } -void ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT& criterial) { - for(auto& pair : criterial) { +void +ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT &criterial) { + for (auto &pair : criterial) { criterias_[pair.first] = pair.second; } } -void ArchiveConf::ParseCritirias(const std::string& criterias) { +void +ArchiveConf::ParseCritirias(const std::string &criterias) { std::stringstream ss(criterias); std::vector tokens; @@ -48,8 +50,8 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) { return; } - for (auto& token : tokens) { - if(token.empty()) { + for (auto &token : tokens) { + if (token.empty()) { continue; } @@ -67,12 +69,12 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) { auto value = std::stoi(kv[1]); criterias_[kv[0]] = value; } - catch (std::out_of_range&){ + catch (std::out_of_range &) { std::string msg = "Out of range: '" + kv[1] + "'"; ENGINE_LOG_ERROR << msg; throw InvalidArgumentException(msg); } - catch (...){ + catch (...) { std::string msg = "Invalid argument: '" + kv[1] + "'"; ENGINE_LOG_ERROR << msg; throw InvalidArgumentException(msg); @@ -80,7 +82,8 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) { } } -void ArchiveConf::ParseType(const std::string& type) { +void +ArchiveConf::ParseType(const std::string &type) { if (type != "delete" && type != "swap") { std::string msg = "Invalid argument: type='" + type + "'"; throw InvalidArgumentException(msg); diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 2ba8c36a32..d8be2767df 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -30,22 +30,27 @@ namespace engine { class Env; -static const char* ARCHIVE_CONF_DISK = "disk"; -static const char* ARCHIVE_CONF_DAYS = "days"; +static const char *ARCHIVE_CONF_DISK = "disk"; +static const char *ARCHIVE_CONF_DAYS = "days"; struct ArchiveConf { using CriteriaT = std::map; - ArchiveConf(const std::string& type, const std::string& criterias = std::string()); + explicit ArchiveConf(const std::string &type, const std::string &criterias = std::string()); - const std::string& GetType() const { return type_; } - const CriteriaT GetCriterias() const { return criterias_; } + const std::string &GetType() const { + return type_; + } - void SetCriterias(const ArchiveConf::CriteriaT& criterial); + const CriteriaT GetCriterias() const { + return criterias_; + } -private: - void ParseCritirias(const std::string& type); - void ParseType(const std::string& criterias); + void SetCriterias(const ArchiveConf::CriteriaT &criterial); + + private: + void ParseCritirias(const std::string &type); + void ParseType(const std::string &criterias); std::string type_; CriteriaT criterias_; @@ -60,12 +65,12 @@ struct DBMetaOptions { struct DBOptions { typedef enum { - SINGLE, - CLUSTER, - READ_ONLY + SINGLE = 0, + CLUSTER_READONLY, + CLUSTER_WRITABLE } MODE; - uint16_t merge_trigger_number_ = 2; + uint16_t merge_trigger_number_ = 2; DBMetaOptions meta_; int mode_ = MODE::SINGLE; diff --git a/cpp/src/db/Types.h b/cpp/src/db/Types.h index d927ecab5f..04bf680937 100644 --- a/cpp/src/db/Types.h +++ b/cpp/src/db/Types.h @@ -21,22 +21,23 @@ #include #include +#include namespace zilliz { namespace milvus { namespace engine { -typedef long IDNumber; -typedef IDNumber* IDNumberPtr; +typedef int64_t IDNumber; +typedef IDNumber *IDNumberPtr; typedef std::vector IDNumbers; typedef std::vector> QueryResult; typedef std::vector QueryResults; struct TableIndex { - int32_t engine_type_ = (int)EngineType::FAISS_IDMAP; + int32_t engine_type_ = (int) EngineType::FAISS_IDMAP; int32_t nlist_ = 16384; - int32_t metric_type_ = (int)MetricType::L2; + int32_t metric_type_ = (int) MetricType::L2; }; } // namespace engine diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index cf61bf6b3d..b3d72202cc 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -15,13 +15,14 @@ // specific language governing permissions and limitations // under the License. -#include "Utils.h" +#include "db/Utils.h" #include "utils/CommonUtil.h" #include "utils/Log.h" #include #include #include +#include #include namespace zilliz { @@ -31,23 +32,25 @@ namespace utils { namespace { -const char* TABLES_FOLDER = "/tables/"; +const char *TABLES_FOLDER = "/tables/"; uint64_t index_file_counter = 0; std::mutex index_file_counter_mutex; -std::string ConstructParentFolder(const std::string& db_path, const meta::TableFileSchema& table_file) { +std::string +ConstructParentFolder(const std::string &db_path, const meta::TableFileSchema &table_file) { std::string table_path = db_path + TABLES_FOLDER + table_file.table_id_; std::string partition_path = table_path + "/" + std::to_string(table_file.date_); return partition_path; } -std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::TableFileSchema& table_file) { +std::string +GetTableFileParentFolder(const DBMetaOptions &options, const meta::TableFileSchema &table_file) { uint64_t path_count = options.slave_paths_.size() + 1; std::string target_path = options.path_; uint64_t index = 0; - if(meta::TableFileSchema::NEW_INDEX == table_file.file_type_) { + if (meta::TableFileSchema::NEW_INDEX == table_file.file_type_) { // index file is large file and to be persisted permanently // we need to distribute index files to each db_path averagely // round robin according to a file counter @@ -67,17 +70,19 @@ std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::T return ConstructParentFolder(target_path, table_file); } -} +} // namespace -long GetMicroSecTimeStamp() { +int64_t +GetMicroSecTimeStamp() { auto now = std::chrono::system_clock::now(); auto micros = std::chrono::duration_cast( - now.time_since_epoch()).count(); + now.time_since_epoch()).count(); return micros; } -Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id) { +Status +CreateTablePath(const DBMetaOptions &options, const std::string &table_id) { std::string db_path = options.path_; std::string table_path = db_path + TABLES_FOLDER + table_id; auto status = server::CommonUtil::CreateDirectory(table_path); @@ -86,7 +91,7 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id return status; } - for(auto& path : options.slave_paths_) { + for (auto &path : options.slave_paths_) { table_path = path + TABLES_FOLDER + table_id; status = server::CommonUtil::CreateDirectory(table_path); if (!status.ok()) { @@ -98,17 +103,18 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id return Status::OK(); } -Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force) { +Status +DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force) { std::vector paths = options.slave_paths_; paths.push_back(options.path_); - for(auto& path : paths) { + for (auto &path : paths) { std::string table_path = path + TABLES_FOLDER + table_id; - if(force) { + if (force) { boost::filesystem::remove_all(table_path); ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; - } else if(boost::filesystem::exists(table_path) && - boost::filesystem::is_empty(table_path)) { + } else if (boost::filesystem::exists(table_path) && + boost::filesystem::is_empty(table_path)) { boost::filesystem::remove_all(table_path); ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; } @@ -117,7 +123,8 @@ Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id return Status::OK(); } -Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { +Status +CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) { std::string parent_path = GetTableFileParentFolder(options, table_file); auto status = server::CommonUtil::CreateDirectory(parent_path); @@ -131,17 +138,18 @@ Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& return Status::OK(); } -Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { +Status +GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) { std::string parent_path = ConstructParentFolder(options.path_, table_file); std::string file_path = parent_path + "/" + table_file.file_id_; - if(boost::filesystem::exists(file_path)) { + if (boost::filesystem::exists(file_path)) { table_file.location_ = file_path; return Status::OK(); } else { - for(auto& path : options.slave_paths_) { + for (auto &path : options.slave_paths_) { parent_path = ConstructParentFolder(path, table_file); file_path = parent_path + "/" + table_file.file_id_; - if(boost::filesystem::exists(file_path)) { + if (boost::filesystem::exists(file_path)) { table_file.location_ = file_path; return Status::OK(); } @@ -155,49 +163,55 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab return Status(DB_ERROR, msg); } -Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { +Status +DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) { utils::GetTableFilePath(options, table_file); boost::filesystem::remove(table_file.location_); return Status::OK(); } -bool IsSameIndex(const TableIndex& index1, const TableIndex& index2) { +bool +IsSameIndex(const TableIndex &index1, const TableIndex &index2) { return index1.engine_type_ == index2.engine_type_ && index1.nlist_ == index2.nlist_ && index1.metric_type_ == index2.metric_type_; } -meta::DateT GetDate(const std::time_t& t, int day_delta) { +meta::DateT +GetDate(const std::time_t &t, int day_delta) { struct tm ltm; localtime_r(&t, <m); if (day_delta > 0) { do { ++ltm.tm_mday; --day_delta; - } while(day_delta > 0); + } while (day_delta > 0); mktime(<m); } else if (day_delta < 0) { do { --ltm.tm_mday; ++day_delta; - } while(day_delta < 0); + } while (day_delta < 0); mktime(<m); } else { ltm.tm_mday; } - return ltm.tm_year*10000 + ltm.tm_mon*100 + ltm.tm_mday; + return ltm.tm_year * 10000 + ltm.tm_mon * 100 + ltm.tm_mday; } -meta::DateT GetDateWithDelta(int day_delta) { +meta::DateT +GetDateWithDelta(int day_delta) { return GetDate(std::time(nullptr), day_delta); } -meta::DateT GetDate() { +meta::DateT +GetDate() { return GetDate(std::time(nullptr), 0); } // URI format: dialect://username:password@host:port/database -Status ParseMetaUri(const std::string& uri, MetaUriInfo& info) { +Status +ParseMetaUri(const std::string &uri, MetaUriInfo &info) { std::string dialect_regex = "(.*)"; std::string username_tegex = "(.*)"; std::string password_regex = "(.*)"; @@ -205,7 +219,7 @@ Status ParseMetaUri(const std::string& uri, MetaUriInfo& info) { std::string port_regex = "(.*)"; std::string db_name_regex = "(.*)"; std::string uri_regex_str = - dialect_regex + "\\:\\/\\/" + + dialect_regex + "\\:\\/\\/" + username_tegex + "\\:" + password_regex + "\\@" + host_regex + "\\:" + diff --git a/cpp/src/db/Utils.h b/cpp/src/db/Utils.h index 39bdfbfbc1..fbeca84686 100644 --- a/cpp/src/db/Utils.h +++ b/cpp/src/db/Utils.h @@ -29,20 +29,30 @@ namespace milvus { namespace engine { namespace utils { -long GetMicroSecTimeStamp(); +int64_t +GetMicroSecTimeStamp(); -Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id); -Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force = true); +Status +CreateTablePath(const DBMetaOptions &options, const std::string &table_id); +Status +DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force = true); -Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); -Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); -Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); +Status +CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file); +Status +GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file); +Status +DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file); -bool IsSameIndex(const TableIndex& index1, const TableIndex& index2); +bool +IsSameIndex(const TableIndex &index1, const TableIndex &index2); -meta::DateT GetDate(const std::time_t &t, int day_delta = 0); -meta::DateT GetDate(); -meta::DateT GetDateWithDelta(int day_delta); +meta::DateT +GetDate(const std::time_t &t, int day_delta = 0); +meta::DateT +GetDate(); +meta::DateT +GetDateWithDelta(int day_delta); struct MetaUriInfo { std::string dialect_; @@ -53,7 +63,8 @@ struct MetaUriInfo { std::string db_name_; }; -Status ParseMetaUri(const std::string& uri, MetaUriInfo& info); +Status +ParseMetaUri(const std::string &uri, MetaUriInfo &info); } // namespace utils } // namespace engine diff --git a/cpp/src/db/engine/EngineFactory.cpp b/cpp/src/db/engine/EngineFactory.cpp index 28cfa97511..043744f803 100644 --- a/cpp/src/db/engine/EngineFactory.cpp +++ b/cpp/src/db/engine/EngineFactory.cpp @@ -15,10 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include "EngineFactory.h" -#include "ExecutionEngineImpl.h" +#include "db/engine/EngineFactory.h" +#include "db/engine/ExecutionEngineImpl.h" #include "utils/Log.h" +#include + namespace zilliz { namespace milvus { namespace engine { @@ -29,20 +31,19 @@ EngineFactory::Build(uint16_t dimension, EngineType index_type, MetricType metric_type, int32_t nlist) { - - if(index_type == EngineType::INVALID) { + if (index_type == EngineType::INVALID) { ENGINE_LOG_ERROR << "Unsupported engine type"; return nullptr; } - ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int)index_type; + ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int) index_type; ExecutionEnginePtr execution_engine_ptr = - std::make_shared(dimension, location, index_type, metric_type, nlist); + std::make_shared(dimension, location, index_type, metric_type, nlist); execution_engine_ptr->Init(); return execution_engine_ptr; } -} -} -} \ No newline at end of file +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/engine/EngineFactory.h b/cpp/src/db/engine/EngineFactory.h index c3fb41adde..105e48f885 100644 --- a/cpp/src/db/engine/EngineFactory.h +++ b/cpp/src/db/engine/EngineFactory.h @@ -21,19 +21,22 @@ #include "ExecutionEngine.h" #include "utils/Status.h" +#include + namespace zilliz { namespace milvus { namespace engine { class EngineFactory { -public: + public: static ExecutionEnginePtr Build(uint16_t dimension, - const std::string& location, + const std::string &location, EngineType index_type, MetricType metric_type, int32_t nlist); }; -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz + diff --git a/cpp/src/db/engine/ExecutionEngine.h b/cpp/src/db/engine/ExecutionEngine.h index 5868f15d33..c63760686f 100644 --- a/cpp/src/db/engine/ExecutionEngine.h +++ b/cpp/src/db/engine/ExecutionEngine.h @@ -21,6 +21,7 @@ #include #include +#include namespace zilliz { namespace milvus { @@ -41,9 +42,8 @@ enum class MetricType { }; class ExecutionEngine { -public: - - virtual Status AddWithIds(long n, const float *xdata, const long *xids) = 0; + public: + virtual Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) = 0; virtual size_t Count() const = 0; @@ -63,16 +63,16 @@ public: virtual std::shared_ptr Clone() = 0; - virtual Status Merge(const std::string& location) = 0; + virtual Status Merge(const std::string &location) = 0; - virtual Status Search(long n, - const float *data, - long k, - long nprobe, - float *distances, - long *labels) const = 0; + virtual Status Search(int64_t n, + const float *data, + int64_t k, + int64_t nprobe, + float *distances, + int64_t *labels) const = 0; - virtual std::shared_ptr BuildIndex(const std::string& location, EngineType engine_type) = 0; + virtual std::shared_ptr BuildIndex(const std::string &location, EngineType engine_type) = 0; virtual Status Cache() = 0; @@ -89,7 +89,6 @@ public: using ExecutionEnginePtr = std::shared_ptr; - } // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index b9d18b4a2f..75b8b5d865 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "ExecutionEngineImpl.h" +#include "db/engine/ExecutionEngineImpl.h" #include "cache/GpuCacheMgr.h" #include "cache/CpuCacheMgr.h" #include "metrics/Metrics.h" @@ -29,6 +29,7 @@ #include "server/Config.h" #include +#include namespace zilliz { namespace milvus { @@ -72,7 +73,8 @@ ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index, nlist_(nlist) { } -VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) { +VecIndexPtr +ExecutionEngineImpl::CreatetVecIndex(EngineType type) { std::shared_ptr index; switch (type) { case EngineType::FAISS_IDMAP: { @@ -99,41 +101,48 @@ VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) { return index; } -Status ExecutionEngineImpl::AddWithIds(long n, const float *xdata, const long *xids) { +Status +ExecutionEngineImpl::AddWithIds(int64_t n, const float *xdata, const int64_t *xids) { auto status = index_->Add(n, xdata, xids); return status; } -size_t ExecutionEngineImpl::Count() const { - if(index_ == nullptr) { +size_t +ExecutionEngineImpl::Count() const { + if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return count 0"; return 0; } return index_->Count(); } -size_t ExecutionEngineImpl::Size() const { +size_t +ExecutionEngineImpl::Size() const { return (size_t) (Count() * Dimension()) * sizeof(float); } -size_t ExecutionEngineImpl::Dimension() const { - if(index_ == nullptr) { +size_t +ExecutionEngineImpl::Dimension() const { + if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return dimension " << dim_; return dim_; } return index_->Dimension(); } -size_t ExecutionEngineImpl::PhysicalSize() const { +size_t +ExecutionEngineImpl::PhysicalSize() const { return server::CommonUtil::GetFileSize(location_); } -Status ExecutionEngineImpl::Serialize() { +Status +ExecutionEngineImpl::Serialize() { auto status = write_index(index_, location_); return status; } -Status ExecutionEngineImpl::Load(bool to_cache) { +Status +ExecutionEngineImpl::Load(bool to_cache) { index_ = cache::CpuCacheMgr::GetInstance()->GetIndex(location_); bool already_in_cache = (index_ != nullptr); if (!already_in_cache) { @@ -141,7 +150,7 @@ Status ExecutionEngineImpl::Load(bool to_cache) { double physical_size = PhysicalSize(); server::CollectExecutionEngineMetrics metrics(physical_size); index_ = read_index(location_); - if(index_ == nullptr) { + if (index_ == nullptr) { std::string msg = "Failed to load index from " + location_; ENGINE_LOG_ERROR << msg; return Status(DB_ERROR, msg); @@ -160,13 +169,14 @@ Status ExecutionEngineImpl::Load(bool to_cache) { return Status::OK(); } -Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) { +Status +ExecutionEngineImpl::CopyToGpu(uint64_t device_id) { auto index = cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_); bool already_in_cache = (index != nullptr); if (already_in_cache) { index_ = index; } else { - if(index_ == nullptr) { + if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to gpu"; return Status(DB_ERROR, "index is null"); } @@ -187,13 +197,14 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) { return Status::OK(); } -Status ExecutionEngineImpl::CopyToCpu() { +Status +ExecutionEngineImpl::CopyToCpu() { auto index = cache::CpuCacheMgr::GetInstance()->GetIndex(location_); bool already_in_cache = (index != nullptr); if (already_in_cache) { index_ = index; } else { - if(index_ == nullptr) { + if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to cpu"; return Status(DB_ERROR, "index is null"); } @@ -213,8 +224,9 @@ Status ExecutionEngineImpl::CopyToCpu() { return Status::OK(); } -ExecutionEnginePtr ExecutionEngineImpl::Clone() { - if(index_ == nullptr) { +ExecutionEnginePtr +ExecutionEngineImpl::Clone() { + if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to clone"; return nullptr; } @@ -225,7 +237,8 @@ ExecutionEnginePtr ExecutionEngineImpl::Clone() { return ret; } -Status ExecutionEngineImpl::Merge(const std::string &location) { +Status +ExecutionEngineImpl::Merge(const std::string &location) { if (location == location_) { return Status(DB_ERROR, "Cannot Merge Self"); } @@ -243,7 +256,7 @@ Status ExecutionEngineImpl::Merge(const std::string &location) { } } - if(index_ == nullptr) { + if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to merge"; return Status(DB_ERROR, "index is null"); } @@ -264,7 +277,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_; auto from_index = std::dynamic_pointer_cast(index_); - if(from_index == nullptr) { + if (from_index == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: from_index is null, failed to build index"; return nullptr; } @@ -282,21 +295,22 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t AutoGenParams(to_index->GetType(), Count(), build_cfg); auto status = to_index->BuildAll(Count(), - from_index->GetRawVectors(), - from_index->GetRawIds(), - build_cfg); + from_index->GetRawVectors(), + from_index->GetRawIds(), + build_cfg); if (!status.ok()) { throw Exception(DB_ERROR, status.message()); } return std::make_shared(to_index, location, engine_type, metric_type_, nlist_); } -Status ExecutionEngineImpl::Search(long n, - const float *data, - long k, - long nprobe, - float *distances, - long *labels) const { - if(index_ == nullptr) { +Status +ExecutionEngineImpl::Search(int64_t n, + const float *data, + int64_t k, + int64_t nprobe, + float *distances, + int64_t *labels) const { + if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search"; return Status(DB_ERROR, "index is null"); } @@ -310,14 +324,16 @@ Status ExecutionEngineImpl::Search(long n, return status; } -Status ExecutionEngineImpl::Cache() { +Status +ExecutionEngineImpl::Cache() { cache::DataObjPtr obj = std::make_shared(index_, PhysicalSize()); zilliz::milvus::cache::CpuCacheMgr::GetInstance()->InsertItem(location_, obj); return Status::OK(); } -Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { +Status +ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { cache::DataObjPtr obj = std::make_shared(index_, PhysicalSize()); zilliz::milvus::cache::GpuCacheMgr::GetInstance(gpu_id)->InsertItem(location_, obj); @@ -325,8 +341,8 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { } // TODO(linxj): remove. -Status ExecutionEngineImpl::Init() { - using namespace zilliz::milvus::server; +Status +ExecutionEngineImpl::Init() { server::Config &config = server::Config::GetInstance(); Status s = config.GetDBConfigBuildIndexGPU(gpu_num_); if (!s.ok()) return s; @@ -334,7 +350,6 @@ Status ExecutionEngineImpl::Init() { return Status::OK(); } - } // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/db/engine/ExecutionEngineImpl.h b/cpp/src/db/engine/ExecutionEngineImpl.h index 1a3c6c83be..6ea09ddb29 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.h +++ b/cpp/src/db/engine/ExecutionEngineImpl.h @@ -23,15 +23,12 @@ #include #include - namespace zilliz { namespace milvus { namespace engine { - class ExecutionEngineImpl : public ExecutionEngine { -public: - + public: ExecutionEngineImpl(uint16_t dimension, const std::string &location, EngineType index_type, @@ -44,7 +41,7 @@ public: MetricType metric_type, int32_t nlist); - Status AddWithIds(long n, const float *xdata, const long *xids) override; + Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) override; size_t Count() const override; @@ -66,12 +63,12 @@ public: Status Merge(const std::string &location) override; - Status Search(long n, + Status Search(int64_t n, const float *data, - long k, - long nprobe, + int64_t k, + int64_t nprobe, float *distances, - long *labels) const override; + int64_t *labels) const override; ExecutionEnginePtr BuildIndex(const std::string &location, EngineType engine_type) override; @@ -81,18 +78,24 @@ public: Status Init() override; - EngineType IndexEngineType() const override { return index_type_; } + EngineType IndexEngineType() const override { + return index_type_; + } - MetricType IndexMetricType() const override { return metric_type_; } + MetricType IndexMetricType() const override { + return metric_type_; + } - std::string GetLocation() const override { return location_; } + std::string GetLocation() const override { + return location_; + } -private: + private: VecIndexPtr CreatetVecIndex(EngineType type); VecIndexPtr Load(const std::string &location); -protected: + protected: VecIndexPtr index_ = nullptr; EngineType index_type_; MetricType metric_type_; @@ -104,7 +107,6 @@ protected: int32_t gpu_num_ = 0; }; - } // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/db/insert/MemManager.h b/cpp/src/db/insert/MemManager.h index 37f0548646..751633bf67 100644 --- a/cpp/src/db/insert/MemManager.h +++ b/cpp/src/db/insert/MemManager.h @@ -23,6 +23,7 @@ #include #include +#include namespace zilliz { namespace milvus { @@ -30,7 +31,6 @@ namespace engine { class MemManager { public: - virtual Status InsertVectors(const std::string &table_id, size_t n, const float *vectors, IDNumbers &vector_ids) = 0; @@ -43,11 +43,10 @@ class MemManager { virtual size_t GetCurrentImmutableMem() = 0; virtual size_t GetCurrentMem() = 0; - }; // MemManagerAbstract using MemManagerPtr = std::shared_ptr; } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/MemManagerImpl.cpp b/cpp/src/db/insert/MemManagerImpl.cpp index ecc14583f9..e555acd28d 100644 --- a/cpp/src/db/insert/MemManagerImpl.cpp +++ b/cpp/src/db/insert/MemManagerImpl.cpp @@ -16,19 +16,19 @@ // under the License. -#include "MemManagerImpl.h" +#include "db/insert/MemManagerImpl.h" #include "VectorSource.h" #include "utils/Log.h" #include "db/Constants.h" #include - namespace zilliz { namespace milvus { namespace engine { -MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) { +MemTablePtr +MemManagerImpl::GetMemByTable(const std::string &table_id) { auto memIt = mem_id_map_.find(table_id); if (memIt != mem_id_map_.end()) { return memIt->second; @@ -38,11 +38,11 @@ MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) { return mem_id_map_[table_id]; } -Status MemManagerImpl::InsertVectors(const std::string &table_id_, - size_t n_, - const float *vectors_, - IDNumbers &vector_ids_) { - +Status +MemManagerImpl::InsertVectors(const std::string &table_id_, + size_t n_, + const float *vectors_, + IDNumbers &vector_ids_) { while (GetCurrentMem() > options_.insert_buffer_size_) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); } @@ -52,11 +52,11 @@ Status MemManagerImpl::InsertVectors(const std::string &table_id_, return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); } -Status MemManagerImpl::InsertVectorsNoLock(const std::string &table_id, - size_t n, - const float *vectors, - IDNumbers &vector_ids) { - +Status +MemManagerImpl::InsertVectorsNoLock(const std::string &table_id, + size_t n, + const float *vectors, + IDNumbers &vector_ids) { MemTablePtr mem = GetMemByTable(table_id); VectorSourcePtr source = std::make_shared(n, vectors); @@ -69,10 +69,11 @@ Status MemManagerImpl::InsertVectorsNoLock(const std::string &table_id, return status; } -Status MemManagerImpl::ToImmutable() { +Status +MemManagerImpl::ToImmutable() { std::unique_lock lock(mutex_); MemIdMap temp_map; - for (auto &kv: mem_id_map_) { + for (auto &kv : mem_id_map_) { if (kv.second->Empty()) { //empty table, no need to serialize temp_map.insert(kv); @@ -85,7 +86,8 @@ Status MemManagerImpl::ToImmutable() { return Status::OK(); } -Status MemManagerImpl::Serialize(std::set &table_ids) { +Status +MemManagerImpl::Serialize(std::set &table_ids) { ToImmutable(); std::unique_lock lock(serialization_mtx_); table_ids.clear(); @@ -97,7 +99,8 @@ Status MemManagerImpl::Serialize(std::set &table_ids) { return Status::OK(); } -Status MemManagerImpl::EraseMemVector(const std::string &table_id) { +Status +MemManagerImpl::EraseMemVector(const std::string &table_id) { {//erase MemVector from rapid-insert cache std::unique_lock lock(mutex_); mem_id_map_.erase(table_id); @@ -117,7 +120,8 @@ Status MemManagerImpl::EraseMemVector(const std::string &table_id) { return Status::OK(); } -size_t MemManagerImpl::GetCurrentMutableMem() { +size_t +MemManagerImpl::GetCurrentMutableMem() { size_t total_mem = 0; for (auto &kv : mem_id_map_) { auto memTable = kv.second; @@ -126,7 +130,8 @@ size_t MemManagerImpl::GetCurrentMutableMem() { return total_mem; } -size_t MemManagerImpl::GetCurrentImmutableMem() { +size_t +MemManagerImpl::GetCurrentImmutableMem() { size_t total_mem = 0; for (auto &mem_table : immu_mem_list_) { total_mem += mem_table->GetCurrentMem(); @@ -134,10 +139,11 @@ size_t MemManagerImpl::GetCurrentImmutableMem() { return total_mem; } -size_t MemManagerImpl::GetCurrentMem() { +size_t +MemManagerImpl::GetCurrentMem() { return GetCurrentMutableMem() + GetCurrentImmutableMem(); } } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/MemManagerImpl.h b/cpp/src/db/insert/MemManagerImpl.h index 767dc0eef0..1783adec18 100644 --- a/cpp/src/db/insert/MemManagerImpl.h +++ b/cpp/src/db/insert/MemManagerImpl.h @@ -24,12 +24,13 @@ #include "utils/Status.h" #include +#include +#include #include #include #include #include - namespace zilliz { namespace milvus { namespace engine { @@ -39,7 +40,8 @@ class MemManagerImpl : public MemManager { using Ptr = std::shared_ptr; MemManagerImpl(const meta::MetaPtr &meta, const DBOptions &options) - : meta_(meta), options_(options) {} + : meta_(meta), options_(options) { + } Status InsertVectors(const std::string &table_id, size_t n, const float *vectors, IDNumbers &vector_ids) override; @@ -71,7 +73,6 @@ class MemManagerImpl : public MemManager { std::mutex serialization_mtx_; }; // NewMemManager - } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/MemMenagerFactory.cpp b/cpp/src/db/insert/MemMenagerFactory.cpp index d0ef51f61f..76e521e168 100644 --- a/cpp/src/db/insert/MemMenagerFactory.cpp +++ b/cpp/src/db/insert/MemMenagerFactory.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "MemMenagerFactory.h" +#include "db/insert/MemMenagerFactory.h" #include "MemManagerImpl.h" #include "utils/Log.h" #include "utils/Exception.h" @@ -26,12 +26,14 @@ #include #include #include +#include namespace zilliz { namespace milvus { namespace engine { -MemManagerPtr MemManagerFactory::Build(const std::shared_ptr& meta, const DBOptions& options) { +MemManagerPtr +MemManagerFactory::Build(const std::shared_ptr &meta, const DBOptions &options) { return std::make_shared(meta, options); } diff --git a/cpp/src/db/insert/MemMenagerFactory.h b/cpp/src/db/insert/MemMenagerFactory.h index d38404f5c0..fa0cb5930a 100644 --- a/cpp/src/db/insert/MemMenagerFactory.h +++ b/cpp/src/db/insert/MemMenagerFactory.h @@ -20,15 +20,17 @@ #include "MemManager.h" #include "db/meta/Meta.h" +#include + namespace zilliz { namespace milvus { namespace engine { class MemManagerFactory { -public: + public: static MemManagerPtr Build(const std::shared_ptr &meta, const DBOptions &options); }; -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTable.cpp b/cpp/src/db/insert/MemTable.cpp index 4be0695572..8db6a9d141 100644 --- a/cpp/src/db/insert/MemTable.cpp +++ b/cpp/src/db/insert/MemTable.cpp @@ -16,9 +16,11 @@ // under the License. -#include "MemTable.h" +#include "db/insert/MemTable.h" #include "utils/Log.h" +#include +#include namespace zilliz { namespace milvus { @@ -30,13 +32,11 @@ MemTable::MemTable(const std::string &table_id, table_id_(table_id), meta_(meta), options_(options) { - } -Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) { - +Status +MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) { while (!source->AllAdded()) { - MemTableFilePtr current_mem_table_file; if (!mem_table_file_list_.empty()) { current_mem_table_file = mem_table_file_list_.back(); @@ -62,15 +62,18 @@ Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) { return Status::OK(); } -void MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) { +void +MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) { mem_table_file = mem_table_file_list_.back(); } -size_t MemTable::GetTableFileCount() { +size_t +MemTable::GetTableFileCount() { return mem_table_file_list_.size(); } -Status MemTable::Serialize() { +Status +MemTable::Serialize() { for (auto mem_table_file = mem_table_file_list_.begin(); mem_table_file != mem_table_file_list_.end();) { auto status = (*mem_table_file)->Serialize(); if (!status.ok()) { @@ -84,15 +87,18 @@ Status MemTable::Serialize() { return Status::OK(); } -bool MemTable::Empty() { +bool +MemTable::Empty() { return mem_table_file_list_.empty(); } -const std::string &MemTable::GetTableId() const { +const std::string & +MemTable::GetTableId() const { return table_id_; } -size_t MemTable::GetCurrentMem() { +size_t +MemTable::GetCurrentMem() { std::lock_guard lock(mutex_); size_t total_mem = 0; for (auto &mem_table_file : mem_table_file_list_) { @@ -103,4 +109,4 @@ size_t MemTable::GetCurrentMem() { } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTable.h b/cpp/src/db/insert/MemTable.h index c1b4125331..da7d914b41 100644 --- a/cpp/src/db/insert/MemTable.h +++ b/cpp/src/db/insert/MemTable.h @@ -23,7 +23,9 @@ #include "utils/Status.h" #include - +#include +#include +#include namespace zilliz { namespace milvus { @@ -59,11 +61,10 @@ class MemTable { DBOptions options_; std::mutex mutex_; - }; //MemTable using MemTablePtr = std::shared_ptr; } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 12dbbcf49f..fc6c2b319a 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -16,14 +16,14 @@ // under the License. -#include "MemTableFile.h" +#include "db/insert/MemTableFile.h" #include "db/Constants.h" #include "db/engine/EngineFactory.h" #include "metrics/Metrics.h" #include "utils/Log.h" #include - +#include namespace zilliz { namespace milvus { @@ -35,20 +35,19 @@ MemTableFile::MemTableFile(const std::string &table_id, table_id_(table_id), meta_(meta), options_(options) { - current_mem_ = 0; auto status = CreateTableFile(); if (status.ok()) { execution_engine_ = EngineFactory::Build(table_file_schema_.dimension_, table_file_schema_.location_, (EngineType) table_file_schema_.engine_type_, - (MetricType)table_file_schema_.metric_type_, + (MetricType) table_file_schema_.metric_type_, table_file_schema_.nlist_); } } -Status MemTableFile::CreateTableFile() { - +Status +MemTableFile::CreateTableFile() { meta::TableFileSchema table_file_schema; table_file_schema.table_id_ = table_id_; auto status = meta_->CreateTableFile(table_file_schema); @@ -61,8 +60,8 @@ Status MemTableFile::CreateTableFile() { return status; } -Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) { - +Status +MemTableFile::Add(const VectorSourcePtr &source, IDNumbers &vector_ids) { if (table_file_schema_.dimension_ <= 0) { std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_; @@ -75,7 +74,8 @@ Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) { if (mem_left >= single_vector_mem_size) { size_t num_vectors_to_add = std::ceil(mem_left / single_vector_mem_size); size_t num_vectors_added; - auto status = source->Add(execution_engine_, table_file_schema_, num_vectors_to_add, num_vectors_added, vector_ids); + auto status = + source->Add(execution_engine_, table_file_schema_, num_vectors_to_add, num_vectors_added, vector_ids); if (status.ok()) { current_mem_ += (num_vectors_added * single_vector_mem_size); } @@ -84,20 +84,24 @@ Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) { return Status::OK(); } -size_t MemTableFile::GetCurrentMem() { +size_t +MemTableFile::GetCurrentMem() { return current_mem_; } -size_t MemTableFile::GetMemLeft() { +size_t +MemTableFile::GetMemLeft() { return (MAX_TABLE_FILE_MEM - current_mem_); } -bool MemTableFile::IsFull() { +bool +MemTableFile::IsFull() { size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; return (GetMemLeft() < single_vector_mem_size); } -Status MemTableFile::Serialize() { +Status +MemTableFile::Serialize() { size_t size = GetCurrentMem(); server::CollectSerializeMetrics metrics(size); @@ -107,7 +111,7 @@ Status MemTableFile::Serialize() { //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size //else set file type to RAW, no need to build index - if (table_file_schema_.engine_type_ != (int)EngineType::FAISS_IDMAP) { + if (table_file_schema_.engine_type_ != (int) EngineType::FAISS_IDMAP) { table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; } else { @@ -117,9 +121,9 @@ Status MemTableFile::Serialize() { auto status = meta_->UpdateTableFile(table_file_schema_); ENGINE_LOG_DEBUG << "New " << ((table_file_schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") - << " file " << table_file_schema_.file_id_ << " of size " << size << " bytes"; + << " file " << table_file_schema_.file_id_ << " of size " << size << " bytes"; - if(options_.insert_cache_immediately_) { + if (options_.insert_cache_immediately_) { execution_engine_->Cache(); } @@ -128,4 +132,4 @@ Status MemTableFile::Serialize() { } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTableFile.h b/cpp/src/db/insert/MemTableFile.h index 168fabe7e9..1d7019a42d 100644 --- a/cpp/src/db/insert/MemTableFile.h +++ b/cpp/src/db/insert/MemTableFile.h @@ -23,17 +23,18 @@ #include "db/engine/ExecutionEngine.h" #include "utils/Status.h" +#include +#include namespace zilliz { namespace milvus { namespace engine { class MemTableFile { - public: MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options); - Status Add(const VectorSourcePtr &source, IDNumbers& vector_ids); + Status Add(const VectorSourcePtr &source, IDNumbers &vector_ids); size_t GetCurrentMem(); @@ -44,25 +45,20 @@ class MemTableFile { Status Serialize(); private: - Status CreateTableFile(); + private: const std::string table_id_; - meta::TableFileSchema table_file_schema_; - meta::MetaPtr meta_; - DBOptions options_; - size_t current_mem_; ExecutionEnginePtr execution_engine_; - }; //MemTableFile using MemTableFilePtr = std::shared_ptr; } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/VectorSource.cpp b/cpp/src/db/insert/VectorSource.cpp index d0aeb5f1ef..330ef08b98 100644 --- a/cpp/src/db/insert/VectorSource.cpp +++ b/cpp/src/db/insert/VectorSource.cpp @@ -16,32 +16,30 @@ // under the License. -#include "VectorSource.h" +#include "db/insert/VectorSource.h" #include "db/engine/ExecutionEngine.h" #include "db/engine/EngineFactory.h" #include "utils/Log.h" #include "metrics/Metrics.h" - namespace zilliz { namespace milvus { namespace engine { - VectorSource::VectorSource(const size_t &n, const float *vectors) : - n_(n), - vectors_(vectors), - id_generator_(std::make_shared()) { + n_(n), + vectors_(vectors), + id_generator_(std::make_shared()) { current_num_vectors_added = 0; } -Status VectorSource::Add(const ExecutionEnginePtr &execution_engine, - const meta::TableFileSchema &table_file_schema, - const size_t &num_vectors_to_add, - size_t &num_vectors_added, - IDNumbers &vector_ids) { - +Status +VectorSource::Add(const ExecutionEnginePtr &execution_engine, + const meta::TableFileSchema &table_file_schema, + const size_t &num_vectors_to_add, + size_t &num_vectors_added, + IDNumbers &vector_ids) { server::CollectAddMetrics metrics(n_, table_file_schema.dimension_); num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ? @@ -52,7 +50,7 @@ Status VectorSource::Add(const ExecutionEnginePtr &execution_engine, } else { vector_ids_to_add.resize(num_vectors_added); for (int pos = current_num_vectors_added; pos < current_num_vectors_added + num_vectors_added; pos++) { - vector_ids_to_add[pos-current_num_vectors_added] = vector_ids[pos]; + vector_ids_to_add[pos - current_num_vectors_added] = vector_ids[pos]; } } Status status = execution_engine->AddWithIds(num_vectors_added, @@ -70,18 +68,21 @@ Status VectorSource::Add(const ExecutionEnginePtr &execution_engine, return status; } -size_t VectorSource::GetNumVectorsAdded() { +size_t +VectorSource::GetNumVectorsAdded() { return current_num_vectors_added; } -bool VectorSource::AllAdded() { +bool +VectorSource::AllAdded() { return (current_num_vectors_added == n_); } -IDNumbers VectorSource::GetVectorIds() { +IDNumbers +VectorSource::GetVectorIds() { return vector_ids_; } } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/VectorSource.h b/cpp/src/db/insert/VectorSource.h index a0292181ce..fd31a14fa6 100644 --- a/cpp/src/db/insert/VectorSource.h +++ b/cpp/src/db/insert/VectorSource.h @@ -23,6 +23,7 @@ #include "db/engine/ExecutionEngine.h" #include "utils/Status.h" +#include namespace zilliz { namespace milvus { @@ -45,7 +46,6 @@ class VectorSource { IDNumbers GetVectorIds(); private: - const size_t n_; const float *vectors_; IDNumbers vector_ids_; @@ -53,11 +53,10 @@ class VectorSource { size_t current_num_vectors_added; std::shared_ptr id_generator_; - }; //VectorSource using VectorSourcePtr = std::shared_ptr; } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index 58168e5fee..42fe6a6dd5 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -25,14 +25,16 @@ #include #include +#include +#include namespace zilliz { namespace milvus { namespace engine { namespace meta { -static const char* META_TABLES = "Tables"; -static const char* META_TABLEFILES = "TableFiles"; +static const char *META_TABLES = "Tables"; +static const char *META_TABLEFILES = "TableFiles"; class Meta { public: @@ -46,7 +48,7 @@ class Meta { virtual Status AllTables(std::vector &table_schema_array) = 0; - virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) = 0; + virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) = 0; virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0; @@ -83,9 +85,9 @@ class Meta { virtual Status FilesByType(const std::string &table_id, const std::vector &file_types, - std::vector& file_ids) = 0; + std::vector &file_ids) = 0; - virtual Status DescribeTableIndex(const std::string &table_id, TableIndex& index) = 0; + virtual Status DescribeTableIndex(const std::string &table_id, TableIndex &index) = 0; virtual Status DropTableIndex(const std::string &table_id) = 0; @@ -96,7 +98,6 @@ class Meta { virtual Status DropAll() = 0; virtual Status Count(const std::string &table_id, uint64_t &result) = 0; - }; // MetaData using MetaPtr = std::shared_ptr; diff --git a/cpp/src/db/meta/MetaConsts.h b/cpp/src/db/meta/MetaConsts.h index 04b548009b..6abae564e8 100644 --- a/cpp/src/db/meta/MetaConsts.h +++ b/cpp/src/db/meta/MetaConsts.h @@ -23,20 +23,20 @@ namespace engine { namespace meta { const size_t K = 1024UL; -const size_t M = K*K; -const size_t G = K*M; -const size_t T = K*G; +const size_t M = K * K; +const size_t G = K * M; +const size_t T = K * G; const size_t S_PS = 1UL; -const size_t MS_PS = 1000*S_PS; -const size_t US_PS = 1000*MS_PS; -const size_t NS_PS = 1000*US_PS; +const size_t MS_PS = 1000 * S_PS; +const size_t US_PS = 1000 * MS_PS; +const size_t NS_PS = 1000 * US_PS; const size_t SECOND = 1UL; -const size_t M_SEC = 60*SECOND; -const size_t H_SEC = 60*M_SEC; -const size_t D_SEC = 24*H_SEC; -const size_t W_SEC = 7*D_SEC; +const size_t M_SEC = 60 * SECOND; +const size_t H_SEC = 60 * M_SEC; +const size_t D_SEC = 24 * H_SEC; +const size_t W_SEC = 7 * D_SEC; } // namespace meta } // namespace engine diff --git a/cpp/src/db/meta/MetaFactory.cpp b/cpp/src/db/meta/MetaFactory.cpp index 8837d54722..dd0b2ae7ea 100644 --- a/cpp/src/db/meta/MetaFactory.cpp +++ b/cpp/src/db/meta/MetaFactory.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "MetaFactory.h" +#include "db/meta/MetaFactory.h" #include "SqliteMetaImpl.h" #include "MySQLMetaImpl.h" #include "utils/Log.h" @@ -28,47 +28,52 @@ #include #include #include +#include namespace zilliz { namespace milvus { namespace engine { - DBMetaOptions MetaFactory::BuildOption(const std::string &path) { - auto p = path; - if(p == "") { - srand(time(nullptr)); - std::stringstream ss; - ss << "/tmp/" << rand(); - p = ss.str(); - } - - DBMetaOptions meta; - meta.path_ = p; - return meta; +DBMetaOptions +MetaFactory::BuildOption(const std::string &path) { + auto p = path; + if (p == "") { + srand(time(nullptr)); + std::stringstream ss; + uint32_t rd = 0; + rand_r(&rd); + ss << "/tmp/" << rd; + p = ss.str(); } - meta::MetaPtr MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) { - std::string uri = metaOptions.backend_uri_; + DBMetaOptions meta; + meta.path_ = p; + return meta; +} - utils::MetaUriInfo uri_info; - auto status = utils::ParseMetaUri(uri, uri_info); - if(!status.ok()) { - ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri; - throw InvalidArgumentException("Wrong URI format "); - } +meta::MetaPtr +MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) { + std::string uri = metaOptions.backend_uri_; - if (strcasecmp(uri_info.dialect_.c_str(), "mysql") == 0) { - ENGINE_LOG_INFO << "Using MySQL"; - return std::make_shared(metaOptions, mode); - } else if (strcasecmp(uri_info.dialect_.c_str(), "sqlite") == 0) { - ENGINE_LOG_INFO << "Using SQLite"; - return std::make_shared(metaOptions); - } else { - ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << uri_info.dialect_; - throw InvalidArgumentException("URI dialect is not mysql / sqlite"); - } + utils::MetaUriInfo uri_info; + auto status = utils::ParseMetaUri(uri, uri_info); + if (!status.ok()) { + ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri; + throw InvalidArgumentException("Wrong URI format "); } + if (strcasecmp(uri_info.dialect_.c_str(), "mysql") == 0) { + ENGINE_LOG_INFO << "Using MySQL"; + return std::make_shared(metaOptions, mode); + } else if (strcasecmp(uri_info.dialect_.c_str(), "sqlite") == 0) { + ENGINE_LOG_INFO << "Using SQLite"; + return std::make_shared(metaOptions); + } else { + ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << uri_info.dialect_; + throw InvalidArgumentException("URI dialect is not mysql / sqlite"); + } +} + } // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/db/meta/MetaFactory.h b/cpp/src/db/meta/MetaFactory.h index edc9706649..a13a09715a 100644 --- a/cpp/src/db/meta/MetaFactory.h +++ b/cpp/src/db/meta/MetaFactory.h @@ -20,18 +20,19 @@ #include "Meta.h" #include "db/Options.h" +#include + namespace zilliz { namespace milvus { namespace engine { class MetaFactory { -public: + public: static DBMetaOptions BuildOption(const std::string &path = ""); static meta::MetaPtr Build(const DBMetaOptions &metaOptions, const int &mode); }; - -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h index cc6dd52057..6fd649bc07 100644 --- a/cpp/src/db/meta/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -23,21 +23,22 @@ #include #include #include +#include namespace zilliz { namespace milvus { namespace engine { namespace meta { -constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP; +constexpr int32_t DEFAULT_ENGINE_TYPE = (int) EngineType::FAISS_IDMAP; constexpr int32_t DEFAULT_NLIST = 16384; -constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2; +constexpr int32_t DEFAULT_METRIC_TYPE = (int) MetricType::L2; constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB; constexpr int64_t FLAG_MASK_NO_USERID = 0x1; -constexpr int64_t FLAG_MASK_HAS_USERID = 0x1<<1; +constexpr int64_t FLAG_MASK_HAS_USERID = 0x1 << 1; -using DateT = int ; +using DateT = int; const DateT EmptyDate = -1; using DatesT = std::vector; @@ -49,7 +50,7 @@ struct TableSchema { size_t id_ = 0; std::string table_id_; - int32_t state_ = (int)NORMAL; + int32_t state_ = (int) NORMAL; uint16_t dimension_ = 0; int64_t created_on_ = 0; int64_t flag_ = 0; diff --git a/cpp/src/db/meta/MySQLConnectionPool.cpp b/cpp/src/db/meta/MySQLConnectionPool.cpp index 87998122f4..9d612d8b54 100644 --- a/cpp/src/db/meta/MySQLConnectionPool.cpp +++ b/cpp/src/db/meta/MySQLConnectionPool.cpp @@ -16,37 +16,39 @@ // under the License. -#include "MySQLConnectionPool.h" +#include "db/meta/MySQLConnectionPool.h" namespace zilliz { namespace milvus { namespace engine { namespace meta { - // Do a simple form of in-use connection limiting: wait to return - // a connection until there are a reasonably low number in use - // already. Can't do this in create() because we're interested in - // connections actually in use, not those created. Also note that - // we keep our own count; ConnectionPool::size() isn't the same! - mysqlpp::Connection *MySQLConnectionPool::grab() { - while (conns_in_use_ > max_pool_size_) { - sleep(1); - } - - ++conns_in_use_; - return mysqlpp::ConnectionPool::grab(); +// Do a simple form of in-use connection limiting: wait to return +// a connection until there are a reasonably low number in use +// already. Can't do this in create() because we're interested in +// connections actually in use, not those created. Also note that +// we keep our own count; ConnectionPool::size() isn't the same! +mysqlpp::Connection * +MySQLConnectionPool::grab() { + while (conns_in_use_ > max_pool_size_) { + sleep(1); } - // Other half of in-use conn count limit - void MySQLConnectionPool::release(const mysqlpp::Connection *pc) { - mysqlpp::ConnectionPool::release(pc); + ++conns_in_use_; + return mysqlpp::ConnectionPool::grab(); +} - if (conns_in_use_ <= 0) { - ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " << conns_in_use_; - } else { - --conns_in_use_; - } +// Other half of in-use conn count limit +void +MySQLConnectionPool::release(const mysqlpp::Connection *pc) { + mysqlpp::ConnectionPool::release(pc); + if (conns_in_use_ <= 0) { + ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " + << conns_in_use_; + } else { + --conns_in_use_; } +} // int MySQLConnectionPool::getConnectionsInUse() { // return conns_in_use_; @@ -56,39 +58,42 @@ namespace meta { // max_idle_time_ = max_idle; // } - std::string MySQLConnectionPool::getDB() { - return db_; - } +std::string +MySQLConnectionPool::getDB() { + return db_; +} - // Superclass overrides - mysqlpp::Connection *MySQLConnectionPool::create() { - - try { - // Create connection using the parameters we were passed upon - // creation. - mysqlpp::Connection *conn = new mysqlpp::Connection(); - conn->set_option(new mysqlpp::ReconnectOption(true)); - conn->connect(db_.empty() ? 0 : db_.c_str(), - server_.empty() ? 0 : server_.c_str(), - user_.empty() ? 0 : user_.c_str(), - password_.empty() ? 0 : password_.c_str(), - port_); - return conn; - } catch (const mysqlpp::ConnectionFailed& er) { - ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what(); - return nullptr; - } +// Superclass overrides +mysqlpp::Connection * +MySQLConnectionPool::create() { + try { + // Create connection using the parameters we were passed upon + // creation. + mysqlpp::Connection *conn = new mysqlpp::Connection(); + conn->set_option(new mysqlpp::ReconnectOption(true)); + conn->connect(db_.empty() ? 0 : db_.c_str(), + server_.empty() ? 0 : server_.c_str(), + user_.empty() ? 0 : user_.c_str(), + password_.empty() ? 0 : password_.c_str(), + port_); + return conn; + } catch (const mysqlpp::ConnectionFailed &er) { + ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what(); + return nullptr; } +} - void MySQLConnectionPool::destroy(mysqlpp::Connection *cp) { - // Our superclass can't know how we created the Connection, so - // it delegates destruction to us, to be safe. - delete cp; - } +void +MySQLConnectionPool::destroy(mysqlpp::Connection *cp) { + // Our superclass can't know how we created the Connection, so + // it delegates destruction to us, to be safe. + delete cp; +} - unsigned int MySQLConnectionPool::max_idle_time() { - return max_idle_time_; - } +unsigned int +MySQLConnectionPool::max_idle_time() { + return max_idle_time_; +} } // namespace meta } // namespace engine diff --git a/cpp/src/db/meta/MySQLConnectionPool.h b/cpp/src/db/meta/MySQLConnectionPool.h index 4aadafd714..761f272bc1 100644 --- a/cpp/src/db/meta/MySQLConnectionPool.h +++ b/cpp/src/db/meta/MySQLConnectionPool.h @@ -30,8 +30,7 @@ namespace engine { namespace meta { class MySQLConnectionPool : public mysqlpp::ConnectionPool { - -public: + public: // The object's only constructor MySQLConnectionPool(std::string dbName, std::string userName, @@ -39,15 +38,13 @@ public: std::string serverIp, int port = 0, int maxPoolSize = 8) : - db_(dbName), - user_(userName), - password_(passWord), - server_(serverIp), - port_(port), - max_pool_size_(maxPoolSize) { - + db_(dbName), + user_(userName), + password_(passWord), + server_(serverIp), + port_(port), + max_pool_size_(maxPoolSize) { conns_in_use_ = 0; - max_idle_time_ = 10; //10 seconds } @@ -68,8 +65,7 @@ public: std::string getDB(); -protected: - + protected: // Superclass overrides mysqlpp::Connection *create() override; @@ -77,7 +73,7 @@ protected: unsigned int max_idle_time() override; -private: + private: // Number of connections currently in use std::atomic conns_in_use_; @@ -93,4 +89,4 @@ private: } // namespace meta } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 48302fa099..490eda1916 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "MySQLMetaImpl.h" +#include "db/meta/MySQLMetaImpl.h" #include "db/IDGenerator.h" #include "db/Utils.h" #include "utils/Log.h" @@ -33,21 +33,21 @@ #include #include #include +#include +#include #include #include - namespace zilliz { namespace milvus { namespace engine { namespace meta { -using namespace mysqlpp; - namespace { -Status HandleException(const std::string &desc, const char* what = nullptr) { - if(what == nullptr) { +Status +HandleException(const std::string &desc, const char *what = nullptr) { + if (what == nullptr) { ENGINE_LOG_ERROR << desc; return Status(DB_META_TRANSACTION_FAILED, desc); } else { @@ -58,8 +58,8 @@ Status HandleException(const std::string &desc, const char* what = nullptr) { } class MetaField { -public: - MetaField(const std::string& name, const std::string& type, const std::string& setting) + public: + MetaField(const std::string &name, const std::string &type, const std::string &setting) : name_(name), type_(type), setting_(setting) { @@ -75,14 +75,14 @@ public: // mysql field type has additional information. for instance, a filed type is defined as 'BIGINT' // we get the type from sql is 'bigint(20)', so we need to ignore the '(20)' - bool IsEqual(const MetaField& field) const { + bool IsEqual(const MetaField &field) const { size_t name_len_min = field.name_.length() > name_.length() ? name_.length() : field.name_.length(); size_t type_len_min = field.type_.length() > type_.length() ? type_.length() : field.type_.length(); return strncasecmp(field.name_.c_str(), name_.c_str(), name_len_min) == 0 && - strncasecmp(field.type_.c_str(), type_.c_str(), type_len_min) == 0; + strncasecmp(field.type_.c_str(), type_.c_str(), type_len_min) == 0; } -private: + private: std::string name_; std::string type_; std::string setting_; @@ -90,8 +90,8 @@ private: using MetaFields = std::vector; class MetaSchema { -public: - MetaSchema(const std::string& name, const MetaFields& fields) + public: + MetaSchema(const std::string &name, const MetaFields &fields) : name_(name), fields_(fields) { } @@ -102,8 +102,8 @@ public: std::string ToString() const { std::string result; - for(auto& field : fields_) { - if(!result.empty()) { + for (auto &field : fields_) { + if (!result.empty()) { result += ","; } result += field.ToString(); @@ -113,11 +113,11 @@ public: //if the outer fields contains all this MetaSchema fields, return true //otherwise return false - bool IsEqual(const MetaFields& fields) const { + bool IsEqual(const MetaFields &fields) const { std::vector found_field; - for(const auto& this_field : fields_) { - for(const auto& outer_field : fields) { - if(this_field.IsEqual(outer_field)) { + for (const auto &this_field : fields_) { + for (const auto &outer_field : fields) { + if (this_field.IsEqual(outer_field)) { found_field.push_back(this_field.name()); break; } @@ -127,7 +127,7 @@ public: return found_field.size() == fields_.size(); } -private: + private: std::string name_; MetaFields fields_; }; @@ -160,20 +160,20 @@ static const MetaSchema TABLEFILES_SCHEMA(META_TABLEFILES, { MetaField("date", "INT", "DEFAULT -1 NOT NULL"), }); -} +} // namespace //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const int &mode) - : options_(options_), +MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options, const int &mode) + : options_(options), mode_(mode) { Initialize(); } MySQLMetaImpl::~MySQLMetaImpl() { - } -Status MySQLMetaImpl::NextTableId(std::string &table_id) { +Status +MySQLMetaImpl::NextTableId(std::string &table_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -181,7 +181,8 @@ Status MySQLMetaImpl::NextTableId(std::string &table_id) { return Status::OK(); } -Status MySQLMetaImpl::NextFileId(std::string &file_id) { +Status +MySQLMetaImpl::NextFileId(std::string &file_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -189,26 +190,27 @@ Status MySQLMetaImpl::NextFileId(std::string &file_id) { return Status::OK(); } -void MySQLMetaImpl::ValidateMetaSchema() { - if(nullptr == mysql_connection_pool_) { +void +MySQLMetaImpl::ValidateMetaSchema() { + if (nullptr == mysql_connection_pool_) { return; } - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return; } - auto validate_func = [&](const MetaSchema& schema) { - Query query_statement = connectionPtr->query(); + auto validate_func = [&](const MetaSchema &schema) { + mysqlpp::Query query_statement = connectionPtr->query(); query_statement << "DESC " << schema.name() << ";"; MetaFields exist_fields; try { - StoreQueryResult res = query_statement.store(); + mysqlpp::StoreQueryResult res = query_statement.store(); for (size_t i = 0; i < res.num_rows(); i++) { - const Row &row = res[i]; + const mysqlpp::Row &row = res[i]; std::string name, type; row["Field"].to_string(name); row["Type"].to_string(type); @@ -219,7 +221,7 @@ void MySQLMetaImpl::ValidateMetaSchema() { ENGINE_LOG_DEBUG << "Meta table '" << schema.name() << "' not exist and will be created"; } - if(exist_fields.empty()) { + if (exist_fields.empty()) { return true; } @@ -237,7 +239,8 @@ void MySQLMetaImpl::ValidateMetaSchema() { } } -Status MySQLMetaImpl::Initialize() { +Status +MySQLMetaImpl::Initialize() { //step 1: create db root path if (!boost::filesystem::is_directory(options_.path_)) { auto ret = boost::filesystem::create_directory(options_.path_); @@ -253,7 +256,7 @@ Status MySQLMetaImpl::Initialize() { //step 2: parse and check meta uri utils::MetaUriInfo uri_info; auto status = utils::ParseMetaUri(uri, uri_info); - if(!status.ok()) { + if (!status.ok()) { std::string msg = "Wrong URI format: " + uri; ENGINE_LOG_ERROR << msg; throw Exception(DB_INVALID_META_URI, msg); @@ -274,8 +277,8 @@ Status MySQLMetaImpl::Initialize() { } mysql_connection_pool_ = - std::make_shared(uri_info.db_name_, uri_info.username_, - uri_info.password_, uri_info.host_, port, max_pool_size); + std::make_shared(uri_info.db_name_, uri_info.username_, + uri_info.password_, uri_info.host_, port, max_pool_size); ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(max_pool_size); //step 4: validate to avoid open old version schema @@ -283,24 +286,22 @@ Status MySQLMetaImpl::Initialize() { //step 5: create meta tables try { - - if (mode_ != DBOptions::MODE::READ_ONLY) { + if (mode_ != DBOptions::MODE::CLUSTER_READONLY) { CleanUp(); } { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - if (!connectionPtr->thread_aware()) { ENGINE_LOG_ERROR << "MySQL++ wasn't built with thread awareness! Can't run without it."; return Status(DB_ERROR, "MySQL++ wasn't built with thread awareness! Can't run without it."); } - Query InitializeQuery = connectionPtr->query(); + mysqlpp::Query InitializeQuery = connectionPtr->query(); InitializeQuery << "CREATE TABLE IF NOT EXISTS " << TABLES_SCHEMA.name() << " (" << TABLES_SCHEMA.ToString() + ");"; @@ -320,7 +321,6 @@ Status MySQLMetaImpl::Initialize() { return HandleException("Initialization Error", InitializeQuery.error()); } } //Scoped Connection - } catch (std::exception &e) { return HandleException("GENERAL ERROR DURING INITIALIZATION", e.what()); } @@ -329,8 +329,9 @@ Status MySQLMetaImpl::Initialize() { } // PXU TODO: Temp solution. Will fix later -Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, - const DatesT &dates) { +Status +MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, + const DatesT &dates) { if (dates.empty()) { return Status::OK(); } @@ -351,59 +352,59 @@ Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", " { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - - Query dropPartitionsByDatesQuery = connectionPtr->query(); + mysqlpp::Query dropPartitionsByDatesQuery = connectionPtr->query(); dropPartitionsByDatesQuery << "UPDATE " << META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "date in (" << dateListStr << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); if (!dropPartitionsByDatesQuery.exec()) { - return HandleException("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", dropPartitionsByDatesQuery.error()); + return HandleException("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", + dropPartitionsByDatesQuery.error()); } } //Scoped Connection ENGINE_LOG_DEBUG << "Successfully drop partitions, table id = " << table_schema.table_id_; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", e.what()); } return Status::OK(); } -Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { +Status +MySQLMetaImpl::CreateTable(TableSchema &table_schema) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query createTableQuery = connectionPtr->query(); + mysqlpp::Query createTableQuery = connectionPtr->query(); if (table_schema.table_id_.empty()) { NextTableId(table_schema.table_id_); } else { createTableQuery << "SELECT state FROM " << META_TABLES << " " << - "WHERE table_id = " << quote << table_schema.table_id_ << ";"; + "WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); - StoreQueryResult res = createTableQuery.store(); + mysqlpp::StoreQueryResult res = createTableQuery.store(); if (res.num_rows() == 1) { int state = res[0]["state"]; @@ -431,13 +432,14 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { createTableQuery << "INSERT INTO " << META_TABLES << " " << - "VALUES(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << - created_on << ", " << flag << ", " << index_file_size << ", " << engine_type << ", " << + "VALUES(" << id << ", " << mysqlpp::quote << table_id << ", " << + state << ", " << dimension << ", " << created_on << ", " << + flag << ", " << index_file_size << ", " << engine_type << ", " << nlist << ", " << metric_type << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); - if (SimpleResult res = createTableQuery.execute()) { + if (mysqlpp::SimpleResult res = createTableQuery.execute()) { table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? //Consume all results to avoid "Commands out of sync" error @@ -448,43 +450,43 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { ENGINE_LOG_DEBUG << "Successfully create table: " << table_schema.table_id_; return utils::CreateTablePath(options_, table_schema.table_id_); - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN CREATING TABLE", e.what()); } } -Status MySQLMetaImpl::FilesByType(const std::string &table_id, - const std::vector &file_types, - std::vector &file_ids) { - if(file_types.empty()) { +Status +MySQLMetaImpl::FilesByType(const std::string &table_id, + const std::vector &file_types, + std::vector &file_ids) { + if (file_types.empty()) { return Status(DB_ERROR, "file types array is empty"); } try { file_ids.clear(); - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } std::string types; - for(auto type : file_types) { - if(!types.empty()) { + for (auto type : file_types) { + if (!types.empty()) { types += ","; } types += std::to_string(type); } - Query hasNonIndexFilesQuery = connectionPtr->query(); + mysqlpp::Query hasNonIndexFilesQuery = connectionPtr->query(); //since table_id is a unique column we just need to check whether it exists or not hasNonIndexFilesQuery << "SELECT file_id, file_type FROM " << META_TABLEFILES << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "file_type in (" << types << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesByType: " << hasNonIndexFilesQuery.str(); @@ -502,29 +504,21 @@ Status MySQLMetaImpl::FilesByType(const std::string &table_id, int32_t file_type = resRow["file_type"]; switch (file_type) { - case (int) TableFileSchema::RAW: - raw_count++; + case (int) TableFileSchema::RAW:raw_count++; break; - case (int) TableFileSchema::NEW: - new_count++; + case (int) TableFileSchema::NEW:new_count++; break; - case (int) TableFileSchema::NEW_MERGE: - new_merge_count++; + case (int) TableFileSchema::NEW_MERGE:new_merge_count++; break; - case (int) TableFileSchema::NEW_INDEX: - new_index_count++; + case (int) TableFileSchema::NEW_INDEX:new_index_count++; break; - case (int) TableFileSchema::TO_INDEX: - to_index_count++; + case (int) TableFileSchema::TO_INDEX:to_index_count++; break; - case (int) TableFileSchema::INDEX: - index_count++; + case (int) TableFileSchema::INDEX:index_count++; break; - case (int) TableFileSchema::BACKUP: - backup_count++; - break; - default: + case (int) TableFileSchema::BACKUP:backup_count++; break; + default:break; } } @@ -533,7 +527,6 @@ Status MySQLMetaImpl::FilesByType(const std::string &table_id, << " new_index files:" << new_index_count << " to_index files:" << to_index_count << " index files:" << index_count << " backup files:" << backup_count; } - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN GET FILE BY TYPE", e.what()); } @@ -541,29 +534,30 @@ Status MySQLMetaImpl::FilesByType(const std::string &table_id, return Status::OK(); } -Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex& index) { +Status +MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex &index) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query updateTableIndexParamQuery = connectionPtr->query(); + mysqlpp::Query updateTableIndexParamQuery = connectionPtr->query(); updateTableIndexParamQuery << "SELECT id, state, dimension, created_on FROM " << META_TABLES << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndex: " << updateTableIndexParamQuery.str(); - StoreQueryResult res = updateTableIndexParamQuery.store(); + mysqlpp::StoreQueryResult res = updateTableIndexParamQuery.store(); if (res.num_rows() == 1) { - const Row &resRow = res[0]; + const mysqlpp::Row &resRow = res[0]; size_t id = resRow["id"]; int32_t state = resRow["state"]; @@ -579,22 +573,20 @@ Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableI "engine_type = " << index.engine_type_ << ", " << "nlist = " << index.nlist_ << ", " << "metric_type = " << index.metric_type_ << " " << - "WHERE table_id = " << quote << table_id << ";"; + "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndex: " << updateTableIndexParamQuery.str(); - if (!updateTableIndexParamQuery.exec()) { - return HandleException("QUERY ERROR WHEN UPDATING TABLE INDEX PARAM", updateTableIndexParamQuery.error()); + return HandleException("QUERY ERROR WHEN UPDATING TABLE INDEX PARAM", + updateTableIndexParamQuery.error()); } } else { return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); } - } //Scoped Connection ENGINE_LOG_DEBUG << "Successfully update table index, table id = " << table_id; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE INDEX PARAM", e.what()); } @@ -602,33 +594,32 @@ Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableI return Status::OK(); } -Status MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { +Status +MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query updateTableFlagQuery = connectionPtr->query(); + mysqlpp::Query updateTableFlagQuery = connectionPtr->query(); updateTableFlagQuery << "UPDATE " << META_TABLES << " " << "SET flag = " << flag << " " << - "WHERE table_id = " << quote << table_id << ";"; + "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFlag: " << updateTableFlagQuery.str(); if (!updateTableFlagQuery.exec()) { return HandleException("QUERY ERROR WHEN UPDATING TABLE FLAG", updateTableFlagQuery.error()); } - } //Scoped Connection ENGINE_LOG_DEBUG << "Successfully update table flag, table id = " << table_id; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); } @@ -636,29 +627,30 @@ Status MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) return Status::OK(); } -Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex& index) { +Status +MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex &index) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query describeTableIndexQuery = connectionPtr->query(); + mysqlpp::Query describeTableIndexQuery = connectionPtr->query(); describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type FROM " << - META_TABLES << " " << - "WHERE table_id = " << quote << table_id << " AND " << - "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + META_TABLES << " " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << + "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTableIndex: " << describeTableIndexQuery.str(); - StoreQueryResult res = describeTableIndexQuery.store(); + mysqlpp::StoreQueryResult res = describeTableIndexQuery.store(); if (res.num_rows() == 1) { - const Row &resRow = res[0]; + const mysqlpp::Row &resRow = res[0]; index.engine_type_ = resRow["engine_type"]; index.nlist_ = resRow["nlist"]; @@ -666,9 +658,7 @@ Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex } else { return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); } - } //Scoped Connection - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); } @@ -676,25 +666,26 @@ Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex return Status::OK(); } -Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { +Status +MySQLMetaImpl::DropTableIndex(const std::string &table_id) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query dropTableIndexQuery = connectionPtr->query(); + mysqlpp::Query dropTableIndexQuery = connectionPtr->query(); //soft delete index files dropTableIndexQuery << "UPDATE " << META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); @@ -708,7 +699,7 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::RAW) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::BACKUP) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); @@ -723,18 +714,16 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { "SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," << "nlist = " << std::to_string(DEFAULT_NLIST) << ", " << "metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " << - "WHERE table_id = " << quote << table_id << ";"; + "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); if (!dropTableIndexQuery.exec()) { return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); } - } //Scoped Connection ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DROPPING TABLE INDEX", e.what()); } @@ -742,38 +731,37 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { return Status::OK(); } -Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { +Status +MySQLMetaImpl::DeleteTable(const std::string &table_id) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } //soft delete table - Query deleteTableQuery = connectionPtr->query(); + mysqlpp::Query deleteTableQuery = connectionPtr->query(); // deleteTableQuery << "UPDATE " << META_TABLES << " " << "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << - "WHERE table_id = " << quote << table_id << ";"; + "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); if (!deleteTableQuery.exec()) { return HandleException("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); } - } //Scoped Connection - if (mode_ == DBOptions::MODE::CLUSTER) { + if (mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { DeleteTableFiles(table_id); } ENGINE_LOG_DEBUG << "Successfully delete table, table id = " << table_id; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DELETING TABLE", e.what()); } @@ -781,24 +769,25 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { return Status::OK(); } -Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { +Status +MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } //soft delete table files - Query deleteTableFilesQuery = connectionPtr->query(); + mysqlpp::Query deleteTableFilesQuery = connectionPtr->query(); // deleteTableFilesQuery << "UPDATE " << META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); @@ -809,7 +798,6 @@ Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { } //Scoped Connection ENGINE_LOG_DEBUG << "Successfully delete table files, table id = " << table_id; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DELETING TABLE FILES", e.what()); } @@ -817,22 +805,24 @@ Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { return Status::OK(); } -Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { +Status +MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { try { server::MetricCollector metric; - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query describeTableQuery = connectionPtr->query(); - describeTableQuery << "SELECT id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type FROM " << - META_TABLES << " " << - "WHERE table_id = " << quote << table_schema.table_id_ << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + mysqlpp::Query describeTableQuery = connectionPtr->query(); + describeTableQuery + << "SELECT id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type " + << " FROM " << META_TABLES << " " << + "WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ << " " << + "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); @@ -840,7 +830,7 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { } //Scoped Connection if (res.num_rows() == 1) { - const Row &resRow = res[0]; + const mysqlpp::Row &resRow = res[0]; table_schema.id_ = resRow["id"]; //implicit conversion @@ -862,7 +852,6 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { } else { return Status(DB_NOT_FOUND, "Table " + table_schema.table_id_ + " not found"); } - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DESCRIBING TABLE", e.what()); } @@ -870,25 +859,26 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { return Status::OK(); } -Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { +Status +MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { try { server::MetricCollector metric; - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query hasTableQuery = connectionPtr->query(); + mysqlpp::Query hasTableQuery = connectionPtr->query(); //since table_id is a unique column we just need to check whether it exists or not hasTableQuery << "SELECT EXISTS " << "(SELECT 1 FROM " << META_TABLES << " " << - "WHERE table_id = " << quote << table_id << " " << + "WHERE table_id = " << mysqlpp::quote << table_id << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << - "AS " << quote << "check" << ";"; + "AS " << mysqlpp::quote << "check" << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); @@ -897,7 +887,6 @@ Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { int check = res[0]["check"]; has_or_not = (check == 1); - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", e.what()); } @@ -905,19 +894,21 @@ Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { return Status::OK(); } -Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { +Status +MySQLMetaImpl::AllTables(std::vector &table_schema_array) { try { server::MetricCollector metric; - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query allTablesQuery = connectionPtr->query(); - allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type FROM " << + mysqlpp::Query allTablesQuery = connectionPtr->query(); + allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type FROM " + << META_TABLES << " " << "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; @@ -954,7 +945,8 @@ Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { return Status::OK(); } -Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { +Status +MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { if (file_schema.date_ == EmptyDate) { file_schema.date_ = utils::GetDate(); } @@ -991,23 +983,24 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { std::string date = std::to_string(file_schema.date_); { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query createTableFileQuery = connectionPtr->query(); + mysqlpp::Query createTableFileQuery = connectionPtr->query(); createTableFileQuery << "INSERT INTO " << META_TABLEFILES << " " << - "VALUES(" << id << ", " << quote << table_id << ", " << engine_type << ", " << - quote << file_id << ", " << file_type << ", " << file_size << ", " << + "VALUES(" << id << ", " << mysqlpp::quote << table_id << + ", " << engine_type << ", " << + mysqlpp::quote << file_id << ", " << file_type << ", " << file_size << ", " << row_count << ", " << updated_time << ", " << created_on << ", " << date << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); - if (SimpleResult res = createTableFileQuery.execute()) { + if (mysqlpp::SimpleResult res = createTableFileQuery.execute()) { file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? //Consume all results to avoid "Commands out of sync" error @@ -1018,29 +1011,31 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { ENGINE_LOG_DEBUG << "Successfully create table file, file id = " << file_schema.file_id_; return utils::CreateTableFilePath(options_, file_schema); - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN CREATING TABLE FILE", e.what()); } } -Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { +Status +MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { files.clear(); try { server::MetricCollector metric; - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query filesToIndexQuery = connectionPtr->query(); - filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " << - META_TABLEFILES << " " << - "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; + mysqlpp::Query filesToIndexQuery = connectionPtr->query(); + filesToIndexQuery + << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " + << + META_TABLEFILES << " " << + "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); @@ -1051,7 +1046,6 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { std::map groups; TableFileSchema table_file; for (auto &resRow : res) { - table_file.id_ = resRow["id"]; //implicit conversion std::string table_id; @@ -1083,7 +1077,6 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { return status; } groups[table_file.table_id_] = table_schema; - } table_file.dimension_ = groups[table_file.table_id_].dimension_; table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_; @@ -1091,43 +1084,44 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { table_file.metric_type_ = groups[table_file.table_id_].metric_type_; auto status = utils::GetTableFilePath(options_, table_file); - if(!status.ok()) { + if (!status.ok()) { ret = status; } files.push_back(table_file); } - if(res.size() > 0) { + if (res.size() > 0) { ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-index files"; } return ret; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", e.what()); } } -Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, - const std::vector &ids, - const DatesT &partition, - DatePartionedTableFilesSchema &files) { +Status +MySQLMetaImpl::FilesToSearch(const std::string &table_id, + const std::vector &ids, + const DatesT &partition, + DatePartionedTableFilesSchema &files) { files.clear(); try { server::MetricCollector metric; - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query filesToSearchQuery = connectionPtr->query(); - filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << quote << table_id; + mysqlpp::Query filesToSearchQuery = connectionPtr->query(); + filesToSearchQuery + << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date FROM " << + META_TABLEFILES << " " << + "WHERE table_id = " << mysqlpp::quote << table_id; if (!partition.empty()) { std::stringstream partitionListSS; @@ -1148,8 +1142,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, std::string idStr = idSS.str(); idStr = idStr.substr(0, idStr.size() - 4); //remove the last " OR " - filesToSearchQuery << " AND " << "(" << idStr << ")"; - + filesToSearchQuery << " AND " << "(" << idStr << ")"; } // End filesToSearchQuery << " AND " << @@ -1172,7 +1165,6 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, Status ret; TableFileSchema table_file; for (auto &resRow : res) { - table_file.id_ = resRow["id"]; //implicit conversion std::string table_id_str; @@ -1202,7 +1194,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, table_file.dimension_ = table_schema.dimension_; auto status = utils::GetTableFilePath(options_, table_file); - if(!status.ok()) { + if (!status.ok()) { ret = status; } @@ -1214,7 +1206,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, files[table_file.date_].push_back(table_file); } - if(res.size() > 0) { + if (res.size() > 0) { ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-search files"; } return ret; @@ -1223,8 +1215,9 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, } } -Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, - DatePartionedTableFilesSchema &files) { +Status +MySQLMetaImpl::FilesToMerge(const std::string &table_id, + DatePartionedTableFilesSchema &files) { files.clear(); try { @@ -1238,20 +1231,22 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, return status; } - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query filesToMergeQuery = connectionPtr->query(); - filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::RAW) << " " << - "ORDER BY row_count DESC" << ";"; + mysqlpp::Query filesToMergeQuery = connectionPtr->query(); + filesToMergeQuery + << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on FROM " + << + META_TABLEFILES << " " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << + "file_type = " << std::to_string(TableFileSchema::RAW) << " " << + "ORDER BY row_count DESC" << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); @@ -1262,7 +1257,7 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, for (auto &resRow : res) { TableFileSchema table_file; table_file.file_size_ = resRow["file_size"]; - if(table_file.file_size_ >= table_schema.index_file_size_) { + if (table_file.file_size_ >= table_schema.index_file_size_) { continue;//skip large file } @@ -1295,7 +1290,7 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, table_file.dimension_ = table_schema.dimension_; auto status = utils::GetTableFilePath(options_, table_file); - if(!status.ok()) { + if (!status.ok()) { ret = status; } @@ -1307,19 +1302,19 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, files[table_file.date_].push_back(table_file); } - if(res.size() > 0) { + if (res.size() > 0) { ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-merge files"; } return ret; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", e.what()); } } -Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, - const std::vector &ids, - TableFilesSchema &table_files) { +Status +MySQLMetaImpl::GetTableFiles(const std::string &table_id, + const std::vector &ids, + TableFilesSchema &table_files) { if (ids.empty()) { return Status::OK(); } @@ -1332,20 +1327,21 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, idStr = idStr.substr(0, idStr.size() - 4); //remove the last " OR " try { - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query getTableFileQuery = connectionPtr->query(); - getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << quote << table_id << " AND " << - "(" << idStr << ") AND " << - "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + mysqlpp::Query getTableFileQuery = connectionPtr->query(); + getTableFileQuery + << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " << + META_TABLEFILES << " " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << + "(" << idStr << ") AND " << + "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); @@ -1358,7 +1354,6 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, Status ret; for (auto &resRow : res) { - TableFileSchema file_schema; file_schema.id_ = resRow["id"]; @@ -1396,14 +1391,14 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, ENGINE_LOG_DEBUG << "Get table files by id"; return ret; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN RETRIEVING TABLE FILES", e.what()); } } // PXU TODO: Support Swap -Status MySQLMetaImpl::Archive() { +Status +MySQLMetaImpl::Archive() { auto &criterias = options_.archive_conf_.GetCriterias(); if (criterias.empty()) { return Status::OK(); @@ -1414,16 +1409,16 @@ Status MySQLMetaImpl::Archive() { auto &limit = kv.second; if (criteria == engine::ARCHIVE_CONF_DAYS) { size_t usecs = limit * D_SEC * US_PS; - long now = utils::GetMicroSecTimeStamp(); + int64_t now = utils::GetMicroSecTimeStamp(); try { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query archiveQuery = connectionPtr->query(); + mysqlpp::Query archiveQuery = connectionPtr->query(); archiveQuery << "UPDATE " << META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << @@ -1437,7 +1432,6 @@ Status MySQLMetaImpl::Archive() { } ENGINE_LOG_DEBUG << "Archive old files"; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DURING ARCHIVE", e.what()); } @@ -1456,19 +1450,20 @@ Status MySQLMetaImpl::Archive() { return Status::OK(); } -Status MySQLMetaImpl::Size(uint64_t &result) { +Status +MySQLMetaImpl::Size(uint64_t &result) { result = 0; try { - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query getSizeQuery = connectionPtr->query(); + mysqlpp::Query getSizeQuery = connectionPtr->query(); getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum FROM " << META_TABLEFILES << " " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; @@ -1483,7 +1478,6 @@ Status MySQLMetaImpl::Size(uint64_t &result) { } else { result = res[0]["sum"]; } - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN RETRIEVING SIZE", e.what()); } @@ -1491,9 +1485,9 @@ Status MySQLMetaImpl::Size(uint64_t &result) { return Status::OK(); } -Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { +Status +MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { if (to_discard_size <= 0) { - return Status::OK(); } ENGINE_LOG_DEBUG << "About to discard size=" << to_discard_size; @@ -1502,13 +1496,13 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { server::MetricCollector metric; bool status; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query discardFilesQuery = connectionPtr->query(); + mysqlpp::Query discardFilesQuery = connectionPtr->query(); discardFilesQuery << "SELECT id, file_size FROM " << META_TABLEFILES << " " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << @@ -1517,7 +1511,7 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); - StoreQueryResult res = discardFilesQuery.store(); + mysqlpp::StoreQueryResult res = discardFilesQuery.store(); if (res.num_rows() == 0) { return Status::OK(); } @@ -1554,36 +1548,36 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { } //Scoped Connection return DiscardFiles(to_discard_size); - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DISCARDING FILES", e.what()); } } //ZR: this function assumes all fields in file_schema have value -Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { +Status +MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query updateTableFileQuery = connectionPtr->query(); + mysqlpp::Query updateTableFileQuery = connectionPtr->query(); //if the table has been deleted, just mark the table file as TO_DELETE //clean thread will delete the file later updateTableFileQuery << "SELECT state FROM " << META_TABLES << " " << - "WHERE table_id = " << quote << file_schema.table_id_ << ";"; + "WHERE table_id = " << mysqlpp::quote << file_schema.table_id_ << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); - StoreQueryResult res = updateTableFileQuery.store(); + mysqlpp::StoreQueryResult res = updateTableFileQuery.store(); if (res.num_rows() == 1) { int state = res[0]["state"]; @@ -1607,9 +1601,9 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { updateTableFileQuery << "UPDATE " << META_TABLEFILES << " " << - "SET table_id = " << quote << table_id << ", " << + "SET table_id = " << mysqlpp::quote << table_id << ", " << "engine_type = " << engine_type << ", " << - "file_id = " << quote << file_id << ", " << + "file_id = " << mysqlpp::quote << file_id << ", " << "file_type = " << file_type << ", " << "file_size = " << file_size << ", " << "row_count = " << row_count << ", " << @@ -1627,7 +1621,6 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { } //Scoped Connection ENGINE_LOG_DEBUG << "Update single table file, file id = " << file_schema.file_id_; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILE", e.what()); } @@ -1635,30 +1628,31 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { return Status::OK(); } -Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { +Status +MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { try { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query updateTableFilesToIndexQuery = connectionPtr->query(); + mysqlpp::Query updateTableFilesToIndexQuery = connectionPtr->query(); updateTableFilesToIndexQuery << "UPDATE " << META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFilesToIndex: " << updateTableFilesToIndexQuery.str(); if (!updateTableFilesToIndexQuery.exec()) { - return HandleException("QUERY ERROR WHEN UPDATING TABLE FILE TO INDEX", updateTableFilesToIndexQuery.error()); + return HandleException("QUERY ERROR WHEN UPDATING TABLE FILE TO INDEX", + updateTableFilesToIndexQuery.error()); } ENGINE_LOG_DEBUG << "Update files to to_index, table id = " << table_id; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX", e.what()); } @@ -1666,21 +1660,21 @@ Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { return Status::OK(); } -Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { +Status +MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query updateTableFilesQuery = connectionPtr->query(); + mysqlpp::Query updateTableFilesQuery = connectionPtr->query(); std::map has_tables; for (auto &file_schema : files) { - if (has_tables.find(file_schema.table_id_) != has_tables.end()) { continue; } @@ -1688,20 +1682,19 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { updateTableFilesQuery << "SELECT EXISTS " << "(SELECT 1 FROM " << META_TABLES << " " << - "WHERE table_id = " << quote << file_schema.table_id_ << " " << + "WHERE table_id = " << mysqlpp::quote << file_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << - "AS " << quote << "check" << ";"; + "AS " << mysqlpp::quote << "check" << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); - StoreQueryResult res = updateTableFilesQuery.store(); + mysqlpp::StoreQueryResult res = updateTableFilesQuery.store(); int check = res[0]["check"]; has_tables[file_schema.table_id_] = (check == 1); } for (auto &file_schema : files) { - if (!has_tables[file_schema.table_id_]) { file_schema.file_type_ = TableFileSchema::TO_DELETE; } @@ -1720,9 +1713,9 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { updateTableFilesQuery << "UPDATE " << META_TABLEFILES << " " << - "SET table_id = " << quote << table_id << ", " << + "SET table_id = " << mysqlpp::quote << table_id << ", " << "engine_type = " << engine_type << ", " << - "file_id = " << quote << file_id << ", " << + "file_id = " << mysqlpp::quote << file_id << ", " << "file_type = " << file_type << ", " << "file_size = " << file_size << ", " << "row_count = " << row_count << ", " << @@ -1740,7 +1733,6 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { } //Scoped Connection ENGINE_LOG_DEBUG << "Update " << files.size() << " table files"; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES", e.what()); } @@ -1748,7 +1740,8 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { return Status::OK(); } -Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { +Status +MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { auto now = utils::GetMicroSecTimeStamp(); std::set table_ids; @@ -1757,13 +1750,13 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date FROM " << META_TABLEFILES << " " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << @@ -1771,13 +1764,12 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); TableFileSchema table_file; std::vector idsToDelete; for (auto &resRow : res) { - table_file.id_ = resRow["id"]; //implicit conversion std::string table_id; @@ -1800,7 +1792,6 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } if (!idsToDelete.empty()) { - std::stringstream idsToDeleteSS; for (auto &id : idsToDelete) { idsToDeleteSS << "id = " << id << " OR "; @@ -1815,16 +1806,15 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); if (!cleanUpFilesWithTTLQuery.exec()) { - return HandleException("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", cleanUpFilesWithTTLQuery.error()); + return HandleException("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", + cleanUpFilesWithTTLQuery.error()); } } - if(res.size() > 0) { + if (res.size() > 0) { ENGINE_LOG_DEBUG << "Clean " << res.size() << " files deleted in " << seconds << " seconds"; } - } //Scoped Connection - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", e.what()); } @@ -1834,23 +1824,22 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); cleanUpFilesWithTTLQuery << "SELECT id, table_id FROM " << META_TABLES << " " << "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); if (!res.empty()) { - std::stringstream idsToDeleteSS; for (auto &resRow : res) { size_t id = resRow["id"]; @@ -1870,15 +1859,15 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); if (!cleanUpFilesWithTTLQuery.exec()) { - return HandleException("QUERY ERROR WHEN CLEANING UP TABLES WITH TTL", cleanUpFilesWithTTLQuery.error()); + return HandleException("QUERY ERROR WHEN CLEANING UP TABLES WITH TTL", + cleanUpFilesWithTTLQuery.error()); } } - if(res.size() > 0) { + if (res.size() > 0) { ENGINE_LOG_DEBUG << "Remove " << res.size() << " tables from meta"; } } //Scoped Connection - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); } @@ -1889,28 +1878,28 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - for(auto& table_id : table_ids) { - Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + for (auto &table_id : table_ids) { + mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); cleanUpFilesWithTTLQuery << "SELECT file_id FROM " << META_TABLEFILES << " " << - "WHERE table_id = " << quote << table_id << ";"; + "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); if (res.empty()) { utils::DeleteTablePath(options_, table_id); } } - if(table_ids.size() > 0) { + if (table_ids.size() > 0) { ENGINE_LOG_DEBUG << "Remove " << table_ids.size() << " tables folder"; } } @@ -1921,30 +1910,31 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return Status::OK(); } -Status MySQLMetaImpl::CleanUp() { +Status +MySQLMetaImpl::CleanUp() { try { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query cleanUpQuery = connectionPtr->query(); + mysqlpp::Query cleanUpQuery = connectionPtr->query(); cleanUpQuery << "SELECT table_name " << "FROM information_schema.tables " << - "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << " " << - "AND table_name = " << quote << META_TABLEFILES << ";"; + "WHERE table_schema = " << mysqlpp::quote << mysql_connection_pool_->getDB() << " " << + "AND table_name = " << mysqlpp::quote << META_TABLEFILES << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); - StoreQueryResult res = cleanUpQuery.store(); + mysqlpp::StoreQueryResult res = cleanUpQuery.store(); if (!res.empty()) { ENGINE_LOG_DEBUG << "Remove table file type as NEW"; cleanUpQuery << "DELETE FROM " << META_TABLEFILES << " WHERE file_type IN (" - << std::to_string(TableFileSchema::NEW) << "," - << std::to_string(TableFileSchema::NEW_MERGE) << "," - << std::to_string(TableFileSchema::NEW_INDEX) << ");"; + << std::to_string(TableFileSchema::NEW) << "," + << std::to_string(TableFileSchema::NEW_MERGE) << "," + << std::to_string(TableFileSchema::NEW_INDEX) << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); @@ -1953,7 +1943,7 @@ Status MySQLMetaImpl::CleanUp() { } } - if(res.size() > 0) { + if (res.size() > 0) { ENGINE_LOG_DEBUG << "Clean " << res.size() << " files"; } } catch (std::exception &e) { @@ -1963,7 +1953,8 @@ Status MySQLMetaImpl::CleanUp() { return Status::OK(); } -Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { +Status +MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { try { server::MetricCollector metric; @@ -1975,19 +1966,18 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { return status; } - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - - Query countQuery = connectionPtr->query(); + mysqlpp::Query countQuery = connectionPtr->query(); countQuery << "SELECT row_count FROM " << META_TABLEFILES << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; @@ -2002,7 +1992,6 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { size_t size = resRow["row_count"]; result += size; } - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN RETRIEVING COUNT", e.what()); } @@ -2010,16 +1999,17 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { return Status::OK(); } -Status MySQLMetaImpl::DropAll() { +Status +MySQLMetaImpl::DropAll() { try { ENGINE_LOG_DEBUG << "Drop all mysql meta"; - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query dropTableQuery = connectionPtr->query(); + mysqlpp::Query dropTableQuery = connectionPtr->query(); dropTableQuery << "DROP TABLE IF EXISTS " << TABLES_SCHEMA.name() << ", " << TABLEFILES_SCHEMA.name() << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index b845b45a8c..d9d67fd748 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -21,26 +21,25 @@ #include "db/Options.h" #include "MySQLConnectionPool.h" -#include "mysql++/mysql++.h" +#include #include - +#include +#include +#include namespace zilliz { namespace milvus { namespace engine { namespace meta { -// auto StoragePrototype(const std::string& path); -using namespace mysqlpp; - class MySQLMetaImpl : public Meta { public: - MySQLMetaImpl(const DBMetaOptions &options_, const int &mode); + MySQLMetaImpl(const DBMetaOptions &options, const int &mode); ~MySQLMetaImpl(); Status CreateTable(TableSchema &table_schema) override; - Status DescribeTable(TableSchema &group_info_) override; + Status DescribeTable(TableSchema &table_schema) override; Status HasTable(const std::string &table_id, bool &has_or_not) override; @@ -63,11 +62,11 @@ class MySQLMetaImpl : public Meta { const std::vector &file_types, std::vector &file_ids) override; - Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) override; + Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override; Status UpdateTableFlag(const std::string &table_id, int64_t flag) override; - Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override; + Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override; Status DropTableIndex(const std::string &table_id) override; @@ -102,12 +101,12 @@ class MySQLMetaImpl : public Meta { private: Status NextFileId(std::string &file_id); Status NextTableId(std::string &table_id); - Status DiscardFiles(long long to_discard_size); + Status DiscardFiles(int64_t to_discard_size); void ValidateMetaSchema(); Status Initialize(); -private: + private: const DBMetaOptions options_; const int mode_; diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 1fe7e3d76c..471b182981 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "SqliteMetaImpl.h" +#include "db/meta/SqliteMetaImpl.h" #include "db/IDGenerator.h" #include "db/Utils.h" #include "utils/Log.h" @@ -29,9 +29,11 @@ #include #include #include +#include +#include +#include #include - namespace zilliz { namespace milvus { namespace engine { @@ -41,8 +43,9 @@ using namespace sqlite_orm; namespace { -Status HandleException(const std::string &desc, const char* what = nullptr) { - if(what == nullptr) { +Status +HandleException(const std::string &desc, const char *what = nullptr) { + if (what == nullptr) { ENGINE_LOG_ERROR << desc; return Status(DB_META_TRANSACTION_FAILED, desc); } else { @@ -52,9 +55,10 @@ Status HandleException(const std::string &desc, const char* what = nullptr) { } } -} +} // namespace -inline auto StoragePrototype(const std::string &path) { +inline auto +StoragePrototype(const std::string &path) { return make_storage(path, make_table(META_TABLES, make_column("id", &TableSchema::id_, primary_key()), @@ -77,24 +81,22 @@ inline auto StoragePrototype(const std::string &path) { make_column("row_count", &TableFileSchema::row_count_, default_value(0)), make_column("updated_time", &TableFileSchema::updated_time_), make_column("created_on", &TableFileSchema::created_on_), - make_column("date", &TableFileSchema::date_)) - ); - + make_column("date", &TableFileSchema::date_))); } using ConnectorT = decltype(StoragePrototype("")); static std::unique_ptr ConnectorPtr; -SqliteMetaImpl::SqliteMetaImpl(const DBMetaOptions &options_) - : options_(options_) { +SqliteMetaImpl::SqliteMetaImpl(const DBMetaOptions &options) + : options_(options) { Initialize(); } SqliteMetaImpl::~SqliteMetaImpl() { - } -Status SqliteMetaImpl::NextTableId(std::string &table_id) { +Status +SqliteMetaImpl::NextTableId(std::string &table_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -102,7 +104,8 @@ Status SqliteMetaImpl::NextTableId(std::string &table_id) { return Status::OK(); } -Status SqliteMetaImpl::NextFileId(std::string &file_id) { +Status +SqliteMetaImpl::NextFileId(std::string &file_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -110,24 +113,26 @@ Status SqliteMetaImpl::NextFileId(std::string &file_id) { return Status::OK(); } -void SqliteMetaImpl::ValidateMetaSchema() { - if(ConnectorPtr == nullptr) { +void +SqliteMetaImpl::ValidateMetaSchema() { + if (ConnectorPtr == nullptr) { return; } //old meta could be recreated since schema changed, throw exception if meta schema is not compatible auto ret = ConnectorPtr->sync_schema_simulate(); - if(ret.find(META_TABLES) != ret.end() - && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) { + if (ret.find(META_TABLES) != ret.end() + && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) { throw Exception(DB_INCOMPATIB_META, "Meta Tables schema is created by Milvus old version"); } - if(ret.find(META_TABLEFILES) != ret.end() - && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) { + if (ret.find(META_TABLEFILES) != ret.end() + && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) { throw Exception(DB_INCOMPATIB_META, "Meta TableFiles schema is created by Milvus old version"); } } -Status SqliteMetaImpl::Initialize() { +Status +SqliteMetaImpl::Initialize() { if (!boost::filesystem::is_directory(options_.path_)) { auto ret = boost::filesystem::create_directory(options_.path_); if (!ret) { @@ -151,8 +156,9 @@ Status SqliteMetaImpl::Initialize() { } // PXU TODO: Temp solution. Will fix later -Status SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id, - const DatesT &dates) { +Status +SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id, + const DatesT &dates) { if (dates.size() == 0) { return Status::OK(); } @@ -171,15 +177,12 @@ Status SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id, ConnectorPtr->update_all( set( c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp() - ), + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), where( c(&TableFileSchema::table_id_) == table_id and - in(&TableFileSchema::date_, dates) - )); + in(&TableFileSchema::date_, dates))); ENGINE_LOG_DEBUG << "Successfully drop partitions, table id = " << table_schema.table_id_; - } catch (std::exception &e) { return HandleException("Encounter exception when drop partition", e.what()); } @@ -187,8 +190,8 @@ Status SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id, return Status::OK(); } -Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) { - +Status +SqliteMetaImpl::CreateTable(TableSchema &table_schema) { try { server::MetricCollector metric; @@ -199,9 +202,9 @@ Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) { NextTableId(table_schema.table_id_); } else { auto table = ConnectorPtr->select(columns(&TableSchema::state_), - where(c(&TableSchema::table_id_) == table_schema.table_id_)); + where(c(&TableSchema::table_id_) == table_schema.table_id_)); if (table.size() == 1) { - if(TableSchema::TO_DELETE == std::get<0>(table[0])) { + if (TableSchema::TO_DELETE == std::get<0>(table[0])) { return Status(DB_ERROR, "Table already exists and it is in delete state, please wait a second"); } else { // Change from no error to already exist. @@ -223,13 +226,13 @@ Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) { ENGINE_LOG_DEBUG << "Successfully create table: " << table_schema.table_id_; return utils::CreateTablePath(options_, table_schema.table_id_); - } catch (std::exception &e) { return HandleException("Encounter exception when create table", e.what()); } } -Status SqliteMetaImpl::DeleteTable(const std::string& table_id) { +Status +SqliteMetaImpl::DeleteTable(const std::string &table_id) { try { server::MetricCollector metric; @@ -238,16 +241,13 @@ Status SqliteMetaImpl::DeleteTable(const std::string& table_id) { //soft delete table ConnectorPtr->update_all( - set( - c(&TableSchema::state_) = (int) TableSchema::TO_DELETE - ), - where( - c(&TableSchema::table_id_) == table_id and - c(&TableSchema::state_) != (int) TableSchema::TO_DELETE - )); + set( + c(&TableSchema::state_) = (int) TableSchema::TO_DELETE), + where( + c(&TableSchema::table_id_) == table_id and + c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); ENGINE_LOG_DEBUG << "Successfully delete table, table id = " << table_id; - } catch (std::exception &e) { return HandleException("Encounter exception when delete table", e.what()); } @@ -255,7 +255,8 @@ Status SqliteMetaImpl::DeleteTable(const std::string& table_id) { return Status::OK(); } -Status SqliteMetaImpl::DeleteTableFiles(const std::string& table_id) { +Status +SqliteMetaImpl::DeleteTableFiles(const std::string &table_id) { try { server::MetricCollector metric; @@ -264,17 +265,14 @@ Status SqliteMetaImpl::DeleteTableFiles(const std::string& table_id) { //soft delete table files ConnectorPtr->update_all( - set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp() - ), - where( - c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE - )); + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); ENGINE_LOG_DEBUG << "Successfully delete table files, table id = " << table_id; - } catch (std::exception &e) { return HandleException("Encounter exception when delete table files", e.what()); } @@ -282,7 +280,8 @@ Status SqliteMetaImpl::DeleteTableFiles(const std::string& table_id) { return Status::OK(); } -Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { +Status +SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { try { server::MetricCollector metric; @@ -296,7 +295,7 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { &TableSchema::nlist_, &TableSchema::metric_type_), where(c(&TableSchema::table_id_) == table_schema.table_id_ - and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); if (groups.size() == 1) { table_schema.id_ = std::get<0>(groups[0]); @@ -311,7 +310,6 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { } else { return Status(DB_NOT_FOUND, "Table " + table_schema.table_id_ + " not found"); } - } catch (std::exception &e) { return HandleException("Encounter exception when describe table", e.what()); } @@ -319,10 +317,11 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { return Status::OK(); } -Status SqliteMetaImpl::FilesByType(const std::string& table_id, - const std::vector& file_types, - std::vector& file_ids) { - if(file_types.empty()) { +Status +SqliteMetaImpl::FilesByType(const std::string &table_id, + const std::vector &file_types, + std::vector &file_ids) { + if (file_types.empty()) { return Status(DB_ERROR, "file types array is empty"); } @@ -331,8 +330,7 @@ Status SqliteMetaImpl::FilesByType(const std::string& table_id, auto selected = ConnectorPtr->select(columns(&TableFileSchema::file_id_, &TableFileSchema::file_type_), where(in(&TableFileSchema::file_type_, file_types) - and c(&TableFileSchema::table_id_) == table_id - )); + and c(&TableFileSchema::table_id_) == table_id)); if (selected.size() >= 1) { int raw_count = 0, new_count = 0, new_merge_count = 0, new_index_count = 0; @@ -340,29 +338,21 @@ Status SqliteMetaImpl::FilesByType(const std::string& table_id, for (auto &file : selected) { file_ids.push_back(std::get<0>(file)); switch (std::get<1>(file)) { - case (int) TableFileSchema::RAW: - raw_count++; + case (int) TableFileSchema::RAW:raw_count++; break; - case (int) TableFileSchema::NEW: - new_count++; + case (int) TableFileSchema::NEW:new_count++; break; - case (int) TableFileSchema::NEW_MERGE: - new_merge_count++; + case (int) TableFileSchema::NEW_MERGE:new_merge_count++; break; - case (int) TableFileSchema::NEW_INDEX: - new_index_count++; + case (int) TableFileSchema::NEW_INDEX:new_index_count++; break; - case (int) TableFileSchema::TO_INDEX: - to_index_count++; + case (int) TableFileSchema::TO_INDEX:to_index_count++; break; - case (int) TableFileSchema::INDEX: - index_count++; + case (int) TableFileSchema::INDEX:index_count++; break; - case (int) TableFileSchema::BACKUP: - backup_count++; - break; - default: + case (int) TableFileSchema::BACKUP:backup_count++; break; + default:break; } } @@ -371,14 +361,14 @@ Status SqliteMetaImpl::FilesByType(const std::string& table_id, << " new_index files:" << new_index_count << " to_index files:" << to_index_count << " index files:" << index_count << " backup files:" << backup_count; } - } catch (std::exception &e) { return HandleException("Encounter exception when check non index files", e.what()); } return Status::OK(); } -Status SqliteMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex& index) { +Status +SqliteMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex &index) { try { server::MetricCollector metric; @@ -392,9 +382,9 @@ Status SqliteMetaImpl::UpdateTableIndex(const std::string &table_id, const Table &TableSchema::flag_, &TableSchema::index_file_size_), where(c(&TableSchema::table_id_) == table_id - and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); - if(tables.size() > 0) { + if (tables.size() > 0) { meta::TableSchema table_schema; table_schema.id_ = std::get<0>(tables[0]); table_schema.table_id_ = table_id; @@ -414,17 +404,14 @@ Status SqliteMetaImpl::UpdateTableIndex(const std::string &table_id, const Table //set all backup file to raw ConnectorPtr->update_all( - set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp() - ), - where( - c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP - )); + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP)); ENGINE_LOG_DEBUG << "Successfully update table index, table id = " << table_id; - } catch (std::exception &e) { std::string msg = "Encounter exception when update table index: table_id = " + table_id; return HandleException(msg, e.what()); @@ -433,20 +420,18 @@ Status SqliteMetaImpl::UpdateTableIndex(const std::string &table_id, const Table return Status::OK(); } -Status SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { +Status +SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { try { server::MetricCollector metric; //set all backup file to raw ConnectorPtr->update_all( - set( - c(&TableSchema::flag_) = flag - ), - where( - c(&TableSchema::table_id_) == table_id - )); + set( + c(&TableSchema::flag_) = flag), + where( + c(&TableSchema::table_id_) == table_id)); ENGINE_LOG_DEBUG << "Successfully update table flag, table id = " << table_id; - } catch (std::exception &e) { std::string msg = "Encounter exception when update table flag: table_id = " + table_id; return HandleException(msg, e.what()); @@ -455,7 +440,8 @@ Status SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag return Status::OK(); } -Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex& index) { +Status +SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex &index) { try { server::MetricCollector metric; @@ -463,7 +449,7 @@ Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableInde &TableSchema::nlist_, &TableSchema::metric_type_), where(c(&TableSchema::table_id_) == table_id - and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); if (groups.size() == 1) { index.engine_type_ = std::get<0>(groups[0]); @@ -472,7 +458,6 @@ Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableInde } else { return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); } - } catch (std::exception &e) { return HandleException("Encounter exception when describe index", e.what()); } @@ -480,7 +465,8 @@ Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableInde return Status::OK(); } -Status SqliteMetaImpl::DropTableIndex(const std::string &table_id) { +Status +SqliteMetaImpl::DropTableIndex(const std::string &table_id) { try { server::MetricCollector metric; @@ -489,39 +475,32 @@ Status SqliteMetaImpl::DropTableIndex(const std::string &table_id) { //soft delete index files ConnectorPtr->update_all( - set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp() - ), - where( - c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::INDEX - )); + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) == (int) TableFileSchema::INDEX)); //set all backup file to raw ConnectorPtr->update_all( - set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp() - ), - where( - c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP - )); + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP)); //set table index type to raw ConnectorPtr->update_all( - set( - c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE, - c(&TableSchema::nlist_) = DEFAULT_NLIST, - c(&TableSchema::metric_type_) = DEFAULT_METRIC_TYPE - ), - where( - c(&TableSchema::table_id_) == table_id - )); + set( + c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE, + c(&TableSchema::nlist_) = DEFAULT_NLIST, + c(&TableSchema::metric_type_) = DEFAULT_METRIC_TYPE), + where( + c(&TableSchema::table_id_) == table_id)); ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; - } catch (std::exception &e) { return HandleException("Encounter exception when delete table index files", e.what()); } @@ -529,20 +508,20 @@ Status SqliteMetaImpl::DropTableIndex(const std::string &table_id) { return Status::OK(); } -Status SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { +Status +SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { has_or_not = false; try { server::MetricCollector metric; auto tables = ConnectorPtr->select(columns(&TableSchema::id_), where(c(&TableSchema::table_id_) == table_id - and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); if (tables.size() == 1) { has_or_not = true; } else { has_or_not = false; } - } catch (std::exception &e) { return HandleException("Encounter exception when lookup table", e.what()); } @@ -550,7 +529,8 @@ Status SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { return Status::OK(); } -Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { +Status +SqliteMetaImpl::AllTables(std::vector &table_schema_array) { try { server::MetricCollector metric; @@ -563,7 +543,7 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { &TableSchema::engine_type_, &TableSchema::nlist_, &TableSchema::metric_type_), - where(c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); + where(c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); for (auto &table : selected) { TableSchema schema; schema.id_ = std::get<0>(table); @@ -578,7 +558,6 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { table_schema_array.emplace_back(schema); } - } catch (std::exception &e) { return HandleException("Encounter exception when lookup all tables", e.what()); } @@ -586,7 +565,8 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { return Status::OK(); } -Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { +Status +SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { if (file_schema.date_ == EmptyDate) { file_schema.date_ = utils::GetDate(); } @@ -619,15 +599,15 @@ Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { ENGINE_LOG_DEBUG << "Successfully create table file, file id = " << file_schema.file_id_; return utils::CreateTableFilePath(options_, file_schema); - - } catch (std::exception& e) { + } catch (std::exception &e) { return HandleException("Encounter exception when create table file", e.what()); } return Status::OK(); } -Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { +Status +SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { files.clear(); try { @@ -661,7 +641,7 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { table_file.created_on_ = std::get<8>(file); auto status = utils::GetTableFilePath(options_, table_file); - if(!status.ok()) { + if (!status.ok()) { ret = status; } auto groupItr = groups.find(table_file.table_id_); @@ -681,20 +661,20 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { files.push_back(table_file); } - if(selected.size() > 0) { + if (selected.size() > 0) { ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-index files"; } return ret; - } catch (std::exception &e) { return HandleException("Encounter exception when iterate raw files", e.what()); } } -Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, - const std::vector &ids, - const DatesT &partition, - DatePartionedTableFilesSchema &files) { +Status +SqliteMetaImpl::FilesToSearch(const std::string &table_id, + const std::vector &ids, + const DatesT &partition, + DatePartionedTableFilesSchema &files) { files.clear(); server::MetricCollector metric; @@ -711,9 +691,9 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, auto match_tableid = c(&TableFileSchema::table_id_) == table_id; std::vector file_types = { - (int) TableFileSchema::RAW, - (int) TableFileSchema::TO_INDEX, - (int) TableFileSchema::INDEX + (int) TableFileSchema::RAW, + (int) TableFileSchema::TO_INDEX, + (int) TableFileSchema::INDEX }; auto match_type = in(&TableFileSchema::file_type_, file_types); @@ -726,18 +706,15 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, if (partition.empty() && ids.empty()) { auto filter = where(match_tableid and match_type); selected = ConnectorPtr->select(select_columns, filter); - } - else if (partition.empty() && !ids.empty()) { + } else if (partition.empty() && !ids.empty()) { auto match_fileid = in(&TableFileSchema::id_, ids); auto filter = where(match_tableid and match_fileid and match_type); selected = ConnectorPtr->select(select_columns, filter); - } - else if (!partition.empty() && ids.empty()) { + } else if (!partition.empty() && ids.empty()) { auto match_date = in(&TableFileSchema::date_, partition); auto filter = where(match_tableid and match_date and match_type); selected = ConnectorPtr->select(select_columns, filter); - } - else if (!partition.empty() && !ids.empty()) { + } else if (!partition.empty() && !ids.empty()) { auto match_fileid = in(&TableFileSchema::id_, ids); auto match_date = in(&TableFileSchema::date_, partition); auto filter = where(match_tableid and match_fileid and match_date and match_type); @@ -761,7 +738,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, table_file.metric_type_ = table_schema.metric_type_; auto status = utils::GetTableFilePath(options_, table_file); - if(!status.ok()) { + if (!status.ok()) { ret = status; } @@ -771,22 +748,22 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, } files[table_file.date_].push_back(table_file); } - if(files.empty()) { + if (files.empty()) { ENGINE_LOG_ERROR << "No file to search for table: " << table_id; } - if(selected.size() > 0) { + if (selected.size() > 0) { ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-search files"; } return ret; - } catch (std::exception &e) { return HandleException("Encounter exception when iterate index files", e.what()); } } -Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, - DatePartionedTableFilesSchema &files) { +Status +SqliteMetaImpl::FilesToMerge(const std::string &table_id, + DatePartionedTableFilesSchema &files) { files.clear(); try { @@ -817,7 +794,7 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, for (auto &file : selected) { TableFileSchema table_file; table_file.file_size_ = std::get<4>(file); - if(table_file.file_size_ >= table_schema.index_file_size_) { + if (table_file.file_size_ >= table_schema.index_file_size_) { continue;//skip large file } @@ -834,7 +811,7 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, table_file.metric_type_ = table_schema.metric_type_; auto status = utils::GetTableFilePath(options_, table_file); - if(!status.ok()) { + if (!status.ok()) { result = status; } @@ -845,19 +822,19 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, files[table_file.date_].push_back(table_file); } - if(selected.size() > 0) { + if (selected.size() > 0) { ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-merge files"; } return result; - } catch (std::exception &e) { return HandleException("Encounter exception when iterate merge files", e.what()); } } -Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, - const std::vector& ids, - TableFilesSchema& table_files) { +Status +SqliteMetaImpl::GetTableFiles(const std::string &table_id, + const std::vector &ids, + TableFilesSchema &table_files) { try { table_files.clear(); auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, @@ -869,9 +846,8 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, &TableFileSchema::engine_type_, &TableFileSchema::created_on_), where(c(&TableFileSchema::table_id_) == table_id and - in(&TableFileSchema::id_, ids) and - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE - )); + in(&TableFileSchema::id_, ids) and + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); TableSchema table_schema; table_schema.table_id_ = table_id; @@ -910,7 +886,8 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, } // PXU TODO: Support Swap -Status SqliteMetaImpl::Archive() { +Status +SqliteMetaImpl::Archive() { auto &criterias = options_.archive_conf_.GetCriterias(); if (criterias.size() == 0) { return Status::OK(); @@ -920,20 +897,18 @@ Status SqliteMetaImpl::Archive() { auto &criteria = kv.first; auto &limit = kv.second; if (criteria == engine::ARCHIVE_CONF_DAYS) { - long usecs = limit * D_SEC * US_PS; - long now = utils::GetMicroSecTimeStamp(); + int64_t usecs = limit * D_SEC * US_PS; + int64_t now = utils::GetMicroSecTimeStamp(); try { //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here std::lock_guard meta_lock(meta_mutex_); ConnectorPtr->update_all( set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE - ), + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE), where( - c(&TableFileSchema::created_on_) < (long) (now - usecs) and - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE - )); + c(&TableFileSchema::created_on_) < (int64_t) (now - usecs) and + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); } catch (std::exception &e) { return HandleException("Encounter exception when update table files", e.what()); } @@ -944,7 +919,7 @@ Status SqliteMetaImpl::Archive() { uint64_t sum = 0; Size(sum); - int64_t to_delete = (int64_t)sum - limit * G; + int64_t to_delete = (int64_t) sum - limit * G; DiscardFiles(to_delete); ENGINE_LOG_DEBUG << "Archive files to free disk"; @@ -954,20 +929,19 @@ Status SqliteMetaImpl::Archive() { return Status::OK(); } -Status SqliteMetaImpl::Size(uint64_t &result) { +Status +SqliteMetaImpl::Size(uint64_t &result) { result = 0; try { auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::file_size_)), - where( - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE - )); + where( + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); for (auto &total_size : selected) { if (!std::get<0>(total_size)) { continue; } result += (uint64_t) (*std::get<0>(total_size)); } - } catch (std::exception &e) { return HandleException("Encounter exception when calculte db size", e.what()); } @@ -975,7 +949,8 @@ Status SqliteMetaImpl::Size(uint64_t &result) { return Status::OK(); } -Status SqliteMetaImpl::DiscardFiles(long to_discard_size) { +Status +SqliteMetaImpl::DiscardFiles(int64_t to_discard_size) { if (to_discard_size <= 0) { return Status::OK(); } @@ -992,7 +967,7 @@ Status SqliteMetaImpl::DiscardFiles(long to_discard_size) { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, &TableFileSchema::file_size_), where(c(&TableFileSchema::file_type_) - != (int) TableFileSchema::TO_DELETE), + != (int) TableFileSchema::TO_DELETE), order_by(&TableFileSchema::id_), limit(10)); @@ -1014,13 +989,11 @@ Status SqliteMetaImpl::DiscardFiles(long to_discard_size) { } ConnectorPtr->update_all( - set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp() - ), - where( - in(&TableFileSchema::id_, ids) - )); + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), + where( + in(&TableFileSchema::id_, ids))); return true; }); @@ -1028,7 +1001,6 @@ Status SqliteMetaImpl::DiscardFiles(long to_discard_size) { if (!commited) { return HandleException("DiscardFiles error: sqlite transaction failed"); } - } catch (std::exception &e) { return HandleException("Encounter exception when discard table file", e.what()); } @@ -1036,7 +1008,8 @@ Status SqliteMetaImpl::DiscardFiles(long to_discard_size) { return DiscardFiles(to_discard_size); } -Status SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { +Status +SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); try { server::MetricCollector metric; @@ -1049,14 +1022,13 @@ Status SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { //if the table has been deleted, just mark the table file as TO_DELETE //clean thread will delete the file later - if(tables.size() < 1 || std::get<0>(tables[0]) == (int)TableSchema::TO_DELETE) { + if (tables.size() < 1 || std::get<0>(tables[0]) == (int) TableSchema::TO_DELETE) { file_schema.file_type_ = TableFileSchema::TO_DELETE; } ConnectorPtr->update(file_schema); ENGINE_LOG_DEBUG << "Update single table file, file id = " << file_schema.file_id_; - } catch (std::exception &e) { std::string msg = "Exception update table file: table_id = " + file_schema.table_id_ + " file_id = " + file_schema.file_id_; @@ -1065,7 +1037,8 @@ Status SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { return Status::OK(); } -Status SqliteMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { +Status +SqliteMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { try { server::MetricCollector metric; @@ -1074,15 +1047,12 @@ Status SqliteMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { ConnectorPtr->update_all( set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_INDEX - ), + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_INDEX), where( c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW - )); + c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW)); ENGINE_LOG_DEBUG << "Update files to to_index, table id = " << table_id; - } catch (std::exception &e) { return HandleException("Encounter exception when update table files to to_index", e.what()); } @@ -1090,7 +1060,8 @@ Status SqliteMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { return Status::OK(); } -Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { +Status +SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { try { server::MetricCollector metric; @@ -1099,13 +1070,13 @@ Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { std::map has_tables; for (auto &file : files) { - if(has_tables.find(file.table_id_) != has_tables.end()) { + if (has_tables.find(file.table_id_) != has_tables.end()) { continue; } auto tables = ConnectorPtr->select(columns(&TableSchema::id_), where(c(&TableSchema::table_id_) == file.table_id_ - and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); - if(tables.size() >= 1) { + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + if (tables.size() >= 1) { has_tables[file.table_id_] = true; } else { has_tables[file.table_id_] = false; @@ -1114,7 +1085,7 @@ Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { auto commited = ConnectorPtr->transaction([&]() mutable { for (auto &file : files) { - if(!has_tables[file.table_id_]) { + if (!has_tables[file.table_id_]) { file.file_type_ = TableFileSchema::TO_DELETE; } @@ -1135,7 +1106,8 @@ Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { return Status::OK(); } -Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { +Status +SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { auto now = utils::GetMicroSecTimeStamp(); std::set table_ids; @@ -1151,11 +1123,11 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { &TableFileSchema::file_id_, &TableFileSchema::date_), where( - c(&TableFileSchema::file_type_) == + c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_DELETE and - c(&TableFileSchema::updated_time_) - < now - seconds * US_PS)); + c(&TableFileSchema::updated_time_) + < now - seconds * US_PS)); auto commited = ConnectorPtr->transaction([&]() mutable { TableFileSchema table_file; @@ -1178,10 +1150,9 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed"); } - if(files.size() > 0) { + if (files.size() > 0) { ENGINE_LOG_DEBUG << "Clean " << files.size() << " files deleted in " << seconds << " seconds"; } - } catch (std::exception &e) { return HandleException("Encounter exception when clean table files", e.what()); } @@ -1210,10 +1181,9 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed"); } - if(tables.size() > 0) { + if (tables.size() > 0) { ENGINE_LOG_DEBUG << "Remove " << tables.size() << " tables from meta"; } - } catch (std::exception &e) { return HandleException("Encounter exception when clean table files", e.what()); } @@ -1223,18 +1193,17 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { try { server::MetricCollector metric; - for(auto& table_id : table_ids) { + for (auto &table_id : table_ids) { auto selected = ConnectorPtr->select(columns(&TableFileSchema::file_id_), where(c(&TableFileSchema::table_id_) == table_id)); - if(selected.size() == 0) { + if (selected.size() == 0) { utils::DeleteTablePath(options_, table_id); } } - if(table_ids.size() > 0) { + if (table_ids.size() > 0) { ENGINE_LOG_DEBUG << "Remove " << table_ids.size() << " tables folder"; } - } catch (std::exception &e) { return HandleException("Encounter exception when delete table folder", e.what()); } @@ -1242,7 +1211,8 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return Status::OK(); } -Status SqliteMetaImpl::CleanUp() { +Status +SqliteMetaImpl::CleanUp() { try { server::MetricCollector metric; @@ -1250,11 +1220,12 @@ Status SqliteMetaImpl::CleanUp() { std::lock_guard meta_lock(meta_mutex_); std::vector file_types = { - (int) TableFileSchema::NEW, - (int) TableFileSchema::NEW_INDEX, - (int) TableFileSchema::NEW_MERGE + (int) TableFileSchema::NEW, + (int) TableFileSchema::NEW_INDEX, + (int) TableFileSchema::NEW_MERGE }; - auto files = ConnectorPtr->select(columns(&TableFileSchema::id_), where(in(&TableFileSchema::file_type_, file_types))); + auto files = + ConnectorPtr->select(columns(&TableFileSchema::id_), where(in(&TableFileSchema::file_type_, file_types))); auto commited = ConnectorPtr->transaction([&]() mutable { for (auto &file : files) { @@ -1268,10 +1239,9 @@ Status SqliteMetaImpl::CleanUp() { return HandleException("CleanUp error: sqlite transaction failed"); } - if(files.size() > 0) { + if (files.size() > 0) { ENGINE_LOG_DEBUG << "Clean " << files.size() << " files"; } - } catch (std::exception &e) { return HandleException("Encounter exception when clean table file", e.what()); } @@ -1279,19 +1249,19 @@ Status SqliteMetaImpl::CleanUp() { return Status::OK(); } -Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) { - +Status +SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) { try { server::MetricCollector metric; std::vector file_types = { - (int) TableFileSchema::RAW, - (int) TableFileSchema::TO_INDEX, - (int) TableFileSchema::INDEX + (int) TableFileSchema::RAW, + (int) TableFileSchema::TO_INDEX, + (int) TableFileSchema::INDEX }; auto selected = ConnectorPtr->select(columns(&TableFileSchema::row_count_), where(in(&TableFileSchema::file_type_, file_types) - and c(&TableFileSchema::table_id_) == table_id)); + and c(&TableFileSchema::table_id_) == table_id)); TableSchema table_schema; table_schema.table_id_ = table_id; @@ -1305,14 +1275,14 @@ Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) { for (auto &file : selected) { result += std::get<0>(file); } - } catch (std::exception &e) { return HandleException("Encounter exception when calculate table file size", e.what()); } return Status::OK(); } -Status SqliteMetaImpl::DropAll() { +Status +SqliteMetaImpl::DropAll() { ENGINE_LOG_DEBUG << "Drop all sqlite meta"; try { diff --git a/cpp/src/db/meta/SqliteMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h index db7a70fe53..978e5e5a4b 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -21,22 +21,25 @@ #include "db/Options.h" #include +#include +#include namespace zilliz { namespace milvus { namespace engine { namespace meta { -auto StoragePrototype(const std::string &path); +auto +StoragePrototype(const std::string &path); class SqliteMetaImpl : public Meta { public: - explicit SqliteMetaImpl(const DBMetaOptions &options_); + explicit SqliteMetaImpl(const DBMetaOptions &options); ~SqliteMetaImpl(); Status CreateTable(TableSchema &table_schema) override; - Status DescribeTable(TableSchema &group_info_) override; + Status DescribeTable(TableSchema &table_schema) override; Status HasTable(const std::string &table_id, bool &has_or_not) override; @@ -58,11 +61,11 @@ class SqliteMetaImpl : public Meta { const std::vector &file_types, std::vector &file_ids) override; - Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) override; + Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override; Status UpdateTableFlag(const std::string &table_id, int64_t flag) override; - Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override; + Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override; Status DropTableIndex(const std::string &table_id) override; @@ -96,12 +99,12 @@ class SqliteMetaImpl : public Meta { private: Status NextFileId(std::string &file_id); Status NextTableId(std::string &table_id); - Status DiscardFiles(long to_discard_size); + Status DiscardFiles(int64_t to_discard_size); void ValidateMetaSchema(); Status Initialize(); -private: + private: const DBMetaOptions options_; std::mutex meta_mutex_; }; // DBMetaImpl diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index dc17733a8b..22c282d4ea 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -27,15 +27,13 @@ #include "utils/CommonUtil.h" #include "metrics/Metrics.h" #include "server/Server.h" -#include "version.h" +#include "../version.h" INITIALIZE_EASYLOGGINGPP void print_help(const std::string &app_name); -using namespace zilliz::milvus; - int main(int argc, char *argv[]) { std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl; @@ -78,7 +76,6 @@ main(int argc, char *argv[]) { std::cout << "Initial log config from: " << log_config_file << std::endl; break; } - case 'p': { char *pid_filename_ptr = strdup(optarg); pid_filename = pid_filename_ptr; @@ -86,7 +83,6 @@ main(int argc, char *argv[]) { std::cout << pid_filename << std::endl; break; } - case 'd': start_daemonized = 1; break; @@ -103,14 +99,14 @@ main(int argc, char *argv[]) { } /* Handle Signal */ - signal(SIGHUP, server::SignalUtil::HandleSignal); - signal(SIGINT, server::SignalUtil::HandleSignal); - signal(SIGUSR1, server::SignalUtil::HandleSignal); - signal(SIGSEGV, server::SignalUtil::HandleSignal); - signal(SIGUSR2, server::SignalUtil::HandleSignal); - signal(SIGTERM, server::SignalUtil::HandleSignal); + signal(SIGHUP, zilliz::milvus::server::SignalUtil::HandleSignal); + signal(SIGINT, zilliz::milvus::server::SignalUtil::HandleSignal); + signal(SIGUSR1, zilliz::milvus::server::SignalUtil::HandleSignal); + signal(SIGSEGV, zilliz::milvus::server::SignalUtil::HandleSignal); + signal(SIGUSR2, zilliz::milvus::server::SignalUtil::HandleSignal); + signal(SIGTERM, zilliz::milvus::server::SignalUtil::HandleSignal); - server::Server &server = server::Server::GetInstance(); + zilliz::milvus::server::Server &server = zilliz::milvus::server::Server::GetInstance(); server.Init(start_daemonized, pid_filename, config_filename, log_config_file); server.Start(); diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index b607dbb020..6f3513072a 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -31,7 +31,7 @@ ErrorCode PrometheusMetrics::Init() { try { Config &config = Config::GetInstance(); - Status s = config.GetMetricConfigAutoBootup(startup_); + Status s = config.GetMetricConfigEnableMonitor(startup_); if (!s.ok()) return s.code(); if (!startup_) return SERVER_SUCCESS; diff --git a/cpp/src/scheduler/Algorithm.cpp b/cpp/src/scheduler/Algorithm.cpp index b388c17bdd..10a585304a 100644 --- a/cpp/src/scheduler/Algorithm.cpp +++ b/cpp/src/scheduler/Algorithm.cpp @@ -16,20 +16,23 @@ // under the License. -#include "Algorithm.h" +#include "scheduler/Algorithm.h" + +#include +#include +#include namespace zilliz { namespace milvus { namespace scheduler { -constexpr uint64_t MAXINT = std::numeric_limits::max(); +constexpr uint64_t MAXINT = std::numeric_limits::max(); uint64_t ShortestPath(const ResourcePtr &src, const ResourcePtr &dest, const ResourceMgrPtr &res_mgr, std::vector &path) { - std::vector> paths; uint64_t num_of_resources = res_mgr->GetAllResources().size(); @@ -53,7 +56,6 @@ ShortestPath(const ResourcePtr &src, std::vector vis(num_of_resources, false); std::vector dis(num_of_resources, MAXINT); for (auto &res : res_mgr->GetAllResources()) { - auto cur_node = std::static_pointer_cast(res); auto cur_neighbours = cur_node->GetNeighbours(); @@ -105,6 +107,6 @@ ShortestPath(const ResourcePtr &src, return dis[name_id_map.at(dest->name())]; } -} -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Algorithm.h b/cpp/src/scheduler/Algorithm.h index be77cf351f..d7e0233ba0 100644 --- a/cpp/src/scheduler/Algorithm.h +++ b/cpp/src/scheduler/Algorithm.h @@ -30,8 +30,8 @@ uint64_t ShortestPath(const ResourcePtr &src, const ResourcePtr &dest, const ResourceMgrPtr &res_mgr, - std::vector& path); + std::vector &path); -} -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Definition.h b/cpp/src/scheduler/Definition.h index 9ac647de88..ce41aca48d 100644 --- a/cpp/src/scheduler/Definition.h +++ b/cpp/src/scheduler/Definition.h @@ -30,7 +30,6 @@ #include "db/engine/EngineFactory.h" #include "db/engine/ExecutionEngine.h" - namespace zilliz { namespace milvus { namespace scheduler { @@ -43,6 +42,6 @@ using EngineFactory = engine::EngineFactory; using EngineType = engine::EngineType; using MetricType = engine::MetricType; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/JobMgr.cpp b/cpp/src/scheduler/JobMgr.cpp index 31db4b344e..0406e98c49 100644 --- a/cpp/src/scheduler/JobMgr.cpp +++ b/cpp/src/scheduler/JobMgr.cpp @@ -15,19 +15,19 @@ // specific language governing permissions and limitations // under the License. -#include "JobMgr.h" +#include "scheduler/JobMgr.h" #include "task/Task.h" #include "TaskCreator.h" +#include namespace zilliz { namespace milvus { namespace scheduler { -using namespace engine; - JobMgr::JobMgr(ResourceMgrPtr res_mgr) - : res_mgr_(std::move(res_mgr)) {} + : res_mgr_(std::move(res_mgr)) { +} void JobMgr::Start() { @@ -59,7 +59,9 @@ void JobMgr::worker_function() { while (running_) { std::unique_lock lock(mutex_); - cv_.wait(lock, [this] { return !queue_.empty(); }); + cv_.wait(lock, [this] { + return !queue_.empty(); + }); auto job = queue_.front(); queue_.pop(); lock.unlock(); @@ -84,6 +86,6 @@ JobMgr::build_task(const JobPtr &job) { return TaskCreator::Create(job); } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/JobMgr.h b/cpp/src/scheduler/JobMgr.h index 8d22b5d4f5..49ba9154e3 100644 --- a/cpp/src/scheduler/JobMgr.h +++ b/cpp/src/scheduler/JobMgr.h @@ -31,15 +31,13 @@ #include "task/Task.h" #include "ResourceMgr.h" - namespace zilliz { namespace milvus { namespace scheduler { class JobMgr { -public: - explicit - JobMgr(ResourceMgrPtr res_mgr); + public: + explicit JobMgr(ResourceMgrPtr res_mgr); void Start(); @@ -47,18 +45,18 @@ public: void Stop(); -public: + public: void Put(const JobPtr &job); -private: + private: void worker_function(); std::vector build_task(const JobPtr &job); -private: + private: bool running_ = false; std::queue queue_; @@ -72,6 +70,6 @@ private: using JobMgrPtr = std::shared_ptr; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceFactory.cpp b/cpp/src/scheduler/ResourceFactory.cpp index 066b47aad7..de9b5bc717 100644 --- a/cpp/src/scheduler/ResourceFactory.cpp +++ b/cpp/src/scheduler/ResourceFactory.cpp @@ -16,8 +16,7 @@ // under the License. -#include "ResourceFactory.h" - +#include "scheduler/ResourceFactory.h" namespace zilliz { namespace milvus { @@ -40,6 +39,6 @@ ResourceFactory::Create(const std::string &name, } } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceFactory.h b/cpp/src/scheduler/ResourceFactory.h index 39fb79154b..f7a47ef1e5 100644 --- a/cpp/src/scheduler/ResourceFactory.h +++ b/cpp/src/scheduler/ResourceFactory.h @@ -25,13 +25,12 @@ #include "resource/GpuResource.h" #include "resource/DiskResource.h" - namespace zilliz { namespace milvus { namespace scheduler { class ResourceFactory { -public: + public: static std::shared_ptr Create(const std::string &name, const std::string &type, @@ -40,8 +39,6 @@ public: bool enable_executor = true); }; - -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceMgr.cpp b/cpp/src/scheduler/ResourceMgr.cpp index 0f9fab8e8d..6067b2eb01 100644 --- a/cpp/src/scheduler/ResourceMgr.cpp +++ b/cpp/src/scheduler/ResourceMgr.cpp @@ -16,15 +16,13 @@ // specific language governing permissions and limitations // under the License. -#include "ResourceMgr.h" +#include "scheduler/ResourceMgr.h" #include "utils/Log.h" - namespace zilliz { namespace milvus { namespace scheduler { - void ResourceMgr::Start() { std::lock_guard lck(resources_mutex_); @@ -186,7 +184,9 @@ void ResourceMgr::event_process() { while (running_) { std::unique_lock lock(event_mutex_); - event_cv_.wait(lock, [this] { return !queue_.empty(); }); + event_cv_.wait(lock, [this] { + return !queue_.empty(); + }); auto event = queue_.front(); queue_.pop(); @@ -201,6 +201,6 @@ ResourceMgr::event_process() { } } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceMgr.h b/cpp/src/scheduler/ResourceMgr.h index 8bd045ab7d..d03408a7df 100644 --- a/cpp/src/scheduler/ResourceMgr.h +++ b/cpp/src/scheduler/ResourceMgr.h @@ -22,21 +22,21 @@ #include #include #include +#include #include #include "resource/Resource.h" #include "utils/Log.h" - namespace zilliz { namespace milvus { namespace scheduler { class ResourceMgr { -public: + public: ResourceMgr() = default; -public: + public: /******** Management Interface ********/ void Start(); @@ -58,7 +58,7 @@ public: subscriber_ = std::move(subscriber); } -public: + public: /******** Management Interface ********/ inline std::vector & GetDiskResources() { @@ -89,10 +89,10 @@ public: uint64_t GetNumGpuResource() const; -public: + public: // TODO: add stats interface(low) -public: + public: /******** Utility Functions ********/ std::string Dump(); @@ -100,14 +100,14 @@ public: std::string DumpTaskTables(); -private: + private: void post_event(const EventPtr &event); void event_process(); -private: + private: bool running_ = false; std::vector disk_resources_; @@ -120,13 +120,11 @@ private: std::condition_variable event_cv_; std::thread worker_thread_; - }; using ResourceMgrPtr = std::shared_ptr; using ResourceMgrWPtr = std::weak_ptr; -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 5d65d35993..71b40de9ee 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -16,12 +16,16 @@ // under the License. -#include "SchedInst.h" +#include "scheduler/SchedInst.h" #include "server/Config.h" #include "ResourceFactory.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "Utils.h" +#include +#include +#include +#include namespace zilliz { namespace milvus { @@ -165,6 +169,7 @@ StopSchedulerService() { SchedInst::GetInstance()->Stop(); ResMgrInst::GetInstance()->Stop(); } -} -} -} + +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/SchedInst.h b/cpp/src/scheduler/SchedInst.h index d23c200b05..4cca6ec5a9 100644 --- a/cpp/src/scheduler/SchedInst.h +++ b/cpp/src/scheduler/SchedInst.h @@ -24,13 +24,12 @@ #include #include - namespace zilliz { namespace milvus { namespace scheduler { class ResMgrInst { -public: + public: static ResourceMgrPtr GetInstance() { if (instance == nullptr) { @@ -42,13 +41,13 @@ public: return instance; } -private: + private: static ResourceMgrPtr instance; static std::mutex mutex_; }; class SchedInst { -public: + public: static SchedulerPtr GetInstance() { if (instance == nullptr) { @@ -60,13 +59,13 @@ public: return instance; } -private: + private: static SchedulerPtr instance; static std::mutex mutex_; }; class JobMgrInst { -public: + public: static scheduler::JobMgrPtr GetInstance() { if (instance == nullptr) { @@ -78,7 +77,7 @@ public: return instance; } -private: + private: static scheduler::JobMgrPtr instance; static std::mutex mutex_; }; @@ -89,6 +88,6 @@ StartSchedulerService(); void StopSchedulerService(); -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Scheduler.cpp b/cpp/src/scheduler/Scheduler.cpp index abaccd4fcb..24f7bfe73b 100644 --- a/cpp/src/scheduler/Scheduler.cpp +++ b/cpp/src/scheduler/Scheduler.cpp @@ -15,13 +15,13 @@ // specific language governing permissions and limitations // under the License. - -#include "src/cache/GpuCacheMgr.h" +#include "scheduler/Scheduler.h" +#include "cache/GpuCacheMgr.h" #include "event/LoadCompletedEvent.h" -#include "Scheduler.h" #include "action/Action.h" #include "Algorithm.h" +#include namespace zilliz { namespace milvus { @@ -43,7 +43,6 @@ Scheduler::Scheduler(ResourceMgrWPtr res_mgr) std::bind(&Scheduler::OnFinishTask, this, std::placeholders::_1))); } - void Scheduler::Start() { running_ = true; @@ -79,7 +78,9 @@ void Scheduler::worker_function() { while (running_) { std::unique_lock lock(event_mutex_); - event_cv_.wait(lock, [this] { return !event_queue_.empty(); }); + event_cv_.wait(lock, [this] { + return !event_queue_.empty(); + }); auto event = event_queue_.front(); event_queue_.pop(); if (event == nullptr) { @@ -142,6 +143,6 @@ Scheduler::OnTaskTableUpdated(const EventPtr &event) { } } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Scheduler.h b/cpp/src/scheduler/Scheduler.h index b591fdd256..073f7eeb0d 100644 --- a/cpp/src/scheduler/Scheduler.h +++ b/cpp/src/scheduler/Scheduler.h @@ -22,22 +22,20 @@ #include #include #include +#include #include "resource/Resource.h" #include "ResourceMgr.h" #include "utils/Log.h" - namespace zilliz { namespace milvus { namespace scheduler { - // TODO: refactor, not friendly to unittest, logical in framework code class Scheduler { -public: - explicit - Scheduler(ResourceMgrWPtr res_mgr); + public: + explicit Scheduler(ResourceMgrWPtr res_mgr); Scheduler(const Scheduler &) = delete; Scheduler(Scheduler &&) = delete; @@ -66,7 +64,7 @@ public: std::string Dump(); -private: + private: /******** Events ********/ /* @@ -106,7 +104,7 @@ private: void OnTaskTableUpdated(const EventPtr &event); -private: + private: /* * Dispatch event to event handler; */ @@ -119,7 +117,7 @@ private: void worker_function(); -private: + private: bool running_; std::unordered_map> event_register_; @@ -133,7 +131,6 @@ private: using SchedulerPtr = std::shared_ptr; -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/TaskCreator.cpp b/cpp/src/scheduler/TaskCreator.cpp index 0d3b2a8a96..8b0378c646 100644 --- a/cpp/src/scheduler/TaskCreator.cpp +++ b/cpp/src/scheduler/TaskCreator.cpp @@ -15,11 +15,10 @@ // specific language governing permissions and limitations // under the License. -#include -#include "TaskCreator.h" +#include "scheduler/TaskCreator.h" +#include "scheduler/tasklabel/BroadcastLabel.h" #include "tasklabel/DefaultLabel.h" - namespace zilliz { namespace milvus { namespace scheduler { @@ -64,8 +63,6 @@ TaskCreator::Create(const DeleteJobPtr &job) { return tasks; } - -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/TaskCreator.h b/cpp/src/scheduler/TaskCreator.h index d4e87bd0a2..81cb25010f 100644 --- a/cpp/src/scheduler/TaskCreator.h +++ b/cpp/src/scheduler/TaskCreator.h @@ -34,17 +34,16 @@ #include "task/SearchTask.h" #include "task/DeleteTask.h" - namespace zilliz { namespace milvus { namespace scheduler { class TaskCreator { -public: + public: static std::vector Create(const JobPtr &job); -public: + public: static std::vector Create(const SearchJobPtr &job); @@ -52,6 +51,6 @@ public: Create(const DeleteJobPtr &job); }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/TaskTable.cpp b/cpp/src/scheduler/TaskTable.cpp index badcb46c2d..a7343ee509 100644 --- a/cpp/src/scheduler/TaskTable.cpp +++ b/cpp/src/scheduler/TaskTable.cpp @@ -16,7 +16,7 @@ // under the License. -#include "TaskTable.h" +#include "scheduler/TaskTable.h" #include "event/TaskTableUpdatedEvent.h" #include "Utils.h" @@ -24,7 +24,6 @@ #include #include - namespace zilliz { namespace milvus { namespace scheduler { @@ -75,6 +74,7 @@ TaskTableItem::Load() { } return false; } + bool TaskTableItem::Loaded() { std::unique_lock lock(mutex); @@ -86,6 +86,7 @@ TaskTableItem::Loaded() { } return false; } + bool TaskTableItem::Execute() { std::unique_lock lock(mutex); @@ -97,6 +98,7 @@ TaskTableItem::Execute() { } return false; } + bool TaskTableItem::Executed() { std::unique_lock lock(mutex); @@ -109,6 +111,7 @@ TaskTableItem::Executed() { } return false; } + bool TaskTableItem::Move() { std::unique_lock lock(mutex); @@ -120,6 +123,7 @@ TaskTableItem::Move() { } return false; } + bool TaskTableItem::Moved() { std::unique_lock lock(mutex); @@ -206,7 +210,6 @@ TaskTable::Put(std::vector &tasks) { } } - TaskTableItemPtr TaskTable::Get(uint64_t index) { return table_[index]; @@ -232,6 +235,6 @@ TaskTable::Dump() { return ss.str(); } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/TaskTable.h b/cpp/src/scheduler/TaskTable.h index c9557c8f9e..819197c1ad 100644 --- a/cpp/src/scheduler/TaskTable.h +++ b/cpp/src/scheduler/TaskTable.h @@ -20,11 +20,13 @@ #include #include #include +#include +#include +#include #include "task/SearchTask.h" #include "event/Event.h" - namespace zilliz { namespace milvus { namespace scheduler { @@ -52,7 +54,8 @@ struct TaskTimestamp { }; struct TaskTableItem { - TaskTableItem() : id(0), task(nullptr), state(TaskTableItemState::INVALID), mutex() {} + TaskTableItem() : id(0), task(nullptr), state(TaskTableItemState::INVALID), mutex() { + } TaskTableItem(const TaskTableItem &src) = delete; TaskTableItem(TaskTableItem &&) = delete; @@ -91,7 +94,7 @@ struct TaskTableItem { using TaskTableItemPtr = std::shared_ptr; class TaskTable { -public: + public: TaskTable() = default; TaskTable(const TaskTable &) = delete; @@ -145,24 +148,28 @@ public: return table_.size(); } -public: + public: TaskTableItemPtr & operator[](uint64_t index) { return table_[index]; } - std::deque::iterator begin() { return table_.begin(); } - std::deque::iterator end() { return table_.end(); } + std::deque::iterator begin() { + return table_.begin(); + } -public: + std::deque::iterator end() { + return table_.end(); + } + + public: std::vector PickToLoad(uint64_t limit); std::vector PickToExecute(uint64_t limit); -public: - + public: /******** Action ********/ // TODO: bool to Status @@ -227,14 +234,14 @@ public: return table_[index]->Moved(); } -public: + public: /* * Dump; */ std::string Dump(); -private: + private: std::uint64_t id_ = 0; mutable std::mutex id_mutex_; std::deque table_; @@ -246,7 +253,6 @@ private: uint64_t last_finish_ = -1; }; - -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Utils.cpp b/cpp/src/scheduler/Utils.cpp index f207845c20..bb19950ffe 100644 --- a/cpp/src/scheduler/Utils.cpp +++ b/cpp/src/scheduler/Utils.cpp @@ -16,12 +16,11 @@ // under the License. -#include "Utils.h" +#include "scheduler/Utils.h" #include #include - namespace zilliz { namespace milvus { namespace scheduler { @@ -41,6 +40,6 @@ get_num_gpu() { return n_devices; } -} -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Utils.h b/cpp/src/scheduler/Utils.h index f1056f104a..c69028f0fa 100644 --- a/cpp/src/scheduler/Utils.h +++ b/cpp/src/scheduler/Utils.h @@ -18,7 +18,6 @@ #include - namespace zilliz { namespace milvus { namespace scheduler { @@ -29,6 +28,6 @@ get_current_timestamp(); uint64_t get_num_gpu(); -} -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/action/Action.h b/cpp/src/scheduler/action/Action.h index 9f2fa9f6c5..a5f67aa98d 100644 --- a/cpp/src/scheduler/action/Action.h +++ b/cpp/src/scheduler/action/Action.h @@ -17,16 +17,17 @@ #pragma once -#include "../resource/Resource.h" -#include "../ResourceMgr.h" +#include "scheduler/resource/Resource.h" +#include "scheduler/ResourceMgr.h" +#include namespace zilliz { namespace milvus { namespace scheduler { class Action { -public: + public: static void PushTaskToNeighbourRandomly(const TaskPtr &task, const ResourcePtr &self); @@ -43,10 +44,8 @@ public: SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr resource, std::shared_ptr event); - }; - -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp index d53903ba2d..909112eb62 100644 --- a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp @@ -22,7 +22,6 @@ #include "src/cache/GpuCacheMgr.h" #include "Action.h" - namespace zilliz { namespace milvus { namespace scheduler { @@ -57,13 +56,12 @@ get_neighbours_with_connetion(const ResourcePtr &self) { return neighbours; } - void Action::PushTaskToNeighbourRandomly(const TaskPtr &task, const ResourcePtr &self) { auto neighbours = get_neighbours_with_connetion(self); if (not neighbours.empty()) { - std::vector speeds; + std::vector speeds; uint64_t total_speed = 0; for (auto &neighbour : neighbours) { uint64_t speed = neighbour.second.speed(); @@ -87,7 +85,6 @@ Action::PushTaskToNeighbourRandomly(const TaskPtr &task, } else { //TODO: process } - } void @@ -99,14 +96,14 @@ Action::PushTaskToAllNeighbour(const TaskPtr &task, const ResourcePtr &self) { } void -Action::PushTaskToResource(const TaskPtr& task, const ResourcePtr& dest) { +Action::PushTaskToResource(const TaskPtr &task, const ResourcePtr &dest) { dest->task_table().Put(task); } void Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, - ResourcePtr resource, - std::shared_ptr event) { + ResourcePtr resource, + std::shared_ptr event) { if (not resource->HasExecutor() && event->task_table_item_->Move()) { auto task = event->task_table_item_->task; auto search_task = std::static_pointer_cast(task); @@ -135,8 +132,8 @@ Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, void Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, - ResourcePtr resource, - std::shared_ptr event) { + ResourcePtr resource, + std::shared_ptr event) { auto task = event->task_table_item_->task; if (resource->type() == ResourceType::DISK) { // step 1: calculate shortest path per resource, from disk to compute resource @@ -181,7 +178,6 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, } } -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/Event.h b/cpp/src/scheduler/event/Event.h index b00bd7f902..860c60c5b7 100644 --- a/cpp/src/scheduler/event/Event.h +++ b/cpp/src/scheduler/event/Event.h @@ -18,6 +18,8 @@ #pragma once #include +#include +#include namespace zilliz { namespace milvus { @@ -33,11 +35,12 @@ enum class EventType { class Resource; class Event { -public: + public: explicit Event(EventType type, std::weak_ptr resource) : type_(type), - resource_(std::move(resource)) {} + resource_(std::move(resource)) { + } inline EventType Type() const { @@ -49,13 +52,13 @@ public: friend std::ostream &operator<<(std::ostream &out, const Event &event); -public: + public: EventType type_; std::weak_ptr resource_; }; using EventPtr = std::shared_ptr; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/EventDump.cpp b/cpp/src/scheduler/event/EventDump.cpp index 3d06dfd5f5..a9ed751d88 100644 --- a/cpp/src/scheduler/event/EventDump.cpp +++ b/cpp/src/scheduler/event/EventDump.cpp @@ -22,36 +22,40 @@ #include "FinishTaskEvent.h" #include "TaskTableUpdatedEvent.h" - namespace zilliz { namespace milvus { namespace scheduler { -std::ostream &operator<<(std::ostream &out, const Event &event) { +std::ostream & +operator<<(std::ostream &out, const Event &event) { out << event.Dump(); return out; } -std::ostream &operator<<(std::ostream &out, const StartUpEvent &event) { +std::ostream & +operator<<(std::ostream &out, const StartUpEvent &event) { out << event.Dump(); return out; } -std::ostream &operator<<(std::ostream &out, const LoadCompletedEvent &event) { +std::ostream & +operator<<(std::ostream &out, const LoadCompletedEvent &event) { out << event.Dump(); return out; } -std::ostream &operator<<(std::ostream &out, const FinishTaskEvent &event) { +std::ostream & +operator<<(std::ostream &out, const FinishTaskEvent &event) { out << event.Dump(); return out; } -std::ostream &operator<<(std::ostream &out, const TaskTableUpdatedEvent &event) { +std::ostream & +operator<<(std::ostream &out, const TaskTableUpdatedEvent &event) { out << event.Dump(); return out; } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/FinishTaskEvent.h b/cpp/src/scheduler/event/FinishTaskEvent.h index 3ffa855f30..f49acb16ad 100644 --- a/cpp/src/scheduler/event/FinishTaskEvent.h +++ b/cpp/src/scheduler/event/FinishTaskEvent.h @@ -17,18 +17,22 @@ #pragma once -#include "Event.h" +#include "scheduler/event/Event.h" +#include +#include +#include namespace zilliz { namespace milvus { namespace scheduler { class FinishTaskEvent : public Event { -public: + public: FinishTaskEvent(std::weak_ptr resource, TaskTableItemPtr task_table_item) : Event(EventType::FINISH_TASK, std::move(resource)), - task_table_item_(std::move(task_table_item)) {} + task_table_item_(std::move(task_table_item)) { + } inline std::string Dump() const override { @@ -37,10 +41,10 @@ public: friend std::ostream &operator<<(std::ostream &out, const FinishTaskEvent &event); -public: + public: TaskTableItemPtr task_table_item_; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/LoadCompletedEvent.h b/cpp/src/scheduler/event/LoadCompletedEvent.h index 286f26723f..8d727f7435 100644 --- a/cpp/src/scheduler/event/LoadCompletedEvent.h +++ b/cpp/src/scheduler/event/LoadCompletedEvent.h @@ -17,19 +17,23 @@ #pragma once -#include "Event.h" -#include "../TaskTable.h" +#include "scheduler/event/Event.h" +#include "scheduler/TaskTable.h" +#include +#include +#include namespace zilliz { namespace milvus { namespace scheduler { class LoadCompletedEvent : public Event { -public: + public: LoadCompletedEvent(std::weak_ptr resource, TaskTableItemPtr task_table_item) : Event(EventType::LOAD_COMPLETED, std::move(resource)), - task_table_item_(std::move(task_table_item)) {} + task_table_item_(std::move(task_table_item)) { + } inline std::string Dump() const override { @@ -38,10 +42,10 @@ public: friend std::ostream &operator<<(std::ostream &out, const LoadCompletedEvent &event); -public: + public: TaskTableItemPtr task_table_item_; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/StartUpEvent.h b/cpp/src/scheduler/event/StartUpEvent.h index 089eb1377b..8e4ad120de 100644 --- a/cpp/src/scheduler/event/StartUpEvent.h +++ b/cpp/src/scheduler/event/StartUpEvent.h @@ -17,18 +17,21 @@ #pragma once -#include "Event.h" +#include "scheduler/event/Event.h" +#include +#include +#include namespace zilliz { namespace milvus { namespace scheduler { class StartUpEvent : public Event { -public: - explicit - StartUpEvent(std::weak_ptr resource) - : Event(EventType::START_UP, std::move(resource)) {} + public: + explicit StartUpEvent(std::weak_ptr resource) + : Event(EventType::START_UP, std::move(resource)) { + } inline std::string Dump() const override { @@ -38,6 +41,6 @@ public: friend std::ostream &operator<<(std::ostream &out, const StartUpEvent &event); }; -} -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/TaskTableUpdatedEvent.h b/cpp/src/scheduler/event/TaskTableUpdatedEvent.h index 70ee28fdab..ec579b31be 100644 --- a/cpp/src/scheduler/event/TaskTableUpdatedEvent.h +++ b/cpp/src/scheduler/event/TaskTableUpdatedEvent.h @@ -19,16 +19,19 @@ #include "Event.h" +#include +#include +#include namespace zilliz { namespace milvus { namespace scheduler { class TaskTableUpdatedEvent : public Event { -public: - explicit - TaskTableUpdatedEvent(std::weak_ptr resource) - : Event(EventType::TASK_TABLE_UPDATED, std::move(resource)) {} + public: + explicit TaskTableUpdatedEvent(std::weak_ptr resource) + : Event(EventType::TASK_TABLE_UPDATED, std::move(resource)) { + } inline std::string Dump() const override { @@ -38,7 +41,6 @@ public: friend std::ostream &operator<<(std::ostream &out, const TaskTableUpdatedEvent &event); }; - -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/DeleteJob.cpp b/cpp/src/scheduler/job/DeleteJob.cpp index 9066afafaa..9d917751c6 100644 --- a/cpp/src/scheduler/job/DeleteJob.cpp +++ b/cpp/src/scheduler/job/DeleteJob.cpp @@ -15,8 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include "DeleteJob.h" +#include "scheduler/job/DeleteJob.h" +#include namespace zilliz { namespace milvus { @@ -29,15 +30,20 @@ DeleteJob::DeleteJob(JobId id, : Job(id, JobType::DELETE), table_id_(std::move(table_id)), meta_ptr_(std::move(meta_ptr)), - num_resource_(num_resource) {} + num_resource_(num_resource) { +} -void DeleteJob::WaitAndDelete() { +void +DeleteJob::WaitAndDelete() { std::unique_lock lock(mutex_); - cv_.wait(lock, [&] { return done_resource == num_resource_; }); + cv_.wait(lock, [&] { + return done_resource == num_resource_; + }); meta_ptr_->DeleteTableFiles(table_id_); } -void DeleteJob::ResourceDone() { +void +DeleteJob::ResourceDone() { { std::lock_guard lock(mutex_); ++done_resource; @@ -45,7 +51,6 @@ void DeleteJob::ResourceDone() { cv_.notify_one(); } -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/DeleteJob.h b/cpp/src/scheduler/job/DeleteJob.h index d82262b235..7d8b20e47c 100644 --- a/cpp/src/scheduler/job/DeleteJob.h +++ b/cpp/src/scheduler/job/DeleteJob.h @@ -30,26 +30,25 @@ #include "Job.h" #include "db/meta/Meta.h" - namespace zilliz { namespace milvus { namespace scheduler { class DeleteJob : public Job { -public: + public: DeleteJob(JobId id, std::string table_id, engine::meta::MetaPtr meta_ptr, uint64_t num_resource); -public: + public: void WaitAndDelete(); void ResourceDone(); -public: + public: std::string table_id() const { return table_id_; @@ -60,7 +59,7 @@ public: return meta_ptr_; } -private: + private: std::string table_id_; engine::meta::MetaPtr meta_ptr_; @@ -72,7 +71,6 @@ private: using DeleteJobPtr = std::shared_ptr; -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/Job.h b/cpp/src/scheduler/job/Job.h index 242c33a4e5..c646a4f034 100644 --- a/cpp/src/scheduler/job/Job.h +++ b/cpp/src/scheduler/job/Job.h @@ -27,7 +27,6 @@ #include #include - namespace zilliz { namespace milvus { namespace scheduler { @@ -42,7 +41,7 @@ enum class JobType { using JobId = std::uint64_t; class Job { -public: + public: inline JobId id() const { return id_; @@ -53,10 +52,11 @@ public: return type_; } -protected: - Job(JobId id, JobType type) : id_(id), type_(type) {} + protected: + Job(JobId id, JobType type) : id_(id), type_(type) { + } -private: + private: JobId id_; JobType type_; }; @@ -64,7 +64,6 @@ private: using JobPtr = std::shared_ptr; using JobWPtr = std::weak_ptr; -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/SearchJob.cpp b/cpp/src/scheduler/job/SearchJob.cpp index 65b6701b5f..dee7125fed 100644 --- a/cpp/src/scheduler/job/SearchJob.cpp +++ b/cpp/src/scheduler/job/SearchJob.cpp @@ -15,11 +15,9 @@ // specific language governing permissions and limitations // under the License. +#include "scheduler/job/SearchJob.h" #include "utils/Log.h" -#include "SearchJob.h" - - namespace zilliz { namespace milvus { namespace scheduler { @@ -33,7 +31,8 @@ SearchJob::SearchJob(zilliz::milvus::scheduler::JobId id, topk_(topk), nq_(nq), nprobe_(nprobe), - vectors_(vectors) {} + vectors_(vectors) { +} bool SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) { @@ -48,11 +47,12 @@ SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) { return true; } - void SearchJob::WaitResult() { std::unique_lock lock(mutex_); - cv_.wait(lock, [this] { return index_files_.empty(); }); + cv_.wait(lock, [this] { + return index_files_.empty(); + }); SERVER_LOG_DEBUG << "SearchJob " << id() << " all done"; } @@ -69,14 +69,11 @@ SearchJob::GetResult() { return result_; } -Status& +Status & SearchJob::GetStatus() { return status_; } - -} -} -} - - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/SearchJob.h b/cpp/src/scheduler/job/SearchJob.h index 12edeab199..7bb7fbefbf 100644 --- a/cpp/src/scheduler/job/SearchJob.h +++ b/cpp/src/scheduler/job/SearchJob.h @@ -26,16 +26,15 @@ #include #include #include +#include #include "Job.h" #include "db/meta/MetaTypes.h" - namespace zilliz { namespace milvus { namespace scheduler { - using engine::meta::TableFileSchemaPtr; using Id2IndexMap = std::unordered_map; @@ -43,10 +42,10 @@ using Id2DistanceMap = std::vector>; using ResultSet = std::vector; class SearchJob : public Job { -public: + public: SearchJob(JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, const float *vectors); -public: + public: bool AddIndexFile(const TableFileSchemaPtr &index_file); @@ -62,7 +61,7 @@ public: Status & GetStatus(); -public: + public: uint64_t topk() const { return topk_; @@ -77,6 +76,7 @@ public: nprobe() const { return nprobe_; } + const float * vectors() const { return vectors_; @@ -87,7 +87,7 @@ public: return index_files_; } -private: + private: uint64_t topk_ = 0; uint64_t nq_ = 0; uint64_t nprobe_ = 0; @@ -105,7 +105,6 @@ private: using SearchJobPtr = std::shared_ptr; -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Connection.h b/cpp/src/scheduler/resource/Connection.h index 38fbc65bb5..cf18b6c9a2 100644 --- a/cpp/src/scheduler/resource/Connection.h +++ b/cpp/src/scheduler/resource/Connection.h @@ -19,17 +19,18 @@ #include #include - +#include namespace zilliz { namespace milvus { namespace scheduler { class Connection { -public: + public: // TODO: update construct function, speed: double->uint64_t Connection(std::string name, double speed) - : name_(std::move(name)), speed_(speed) {} + : name_(std::move(name)), speed_(speed) { + } const std::string & name() const { @@ -46,7 +47,7 @@ public: return 1024 / speed_; } -public: + public: std::string Dump() const { std::stringstream ss; @@ -54,12 +55,11 @@ public: return ss.str(); } -private: + private: std::string name_; uint64_t speed_; }; - -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/CpuResource.cpp b/cpp/src/scheduler/resource/CpuResource.cpp index 3933bbb5a0..5859dfd0cd 100644 --- a/cpp/src/scheduler/resource/CpuResource.cpp +++ b/cpp/src/scheduler/resource/CpuResource.cpp @@ -16,29 +16,34 @@ // under the License. -#include "CpuResource.h" +#include "scheduler/resource/CpuResource.h" +#include namespace zilliz { namespace milvus { namespace scheduler { -std::ostream &operator<<(std::ostream &out, const CpuResource &resource) { +std::ostream & +operator<<(std::ostream &out, const CpuResource &resource) { out << resource.Dump(); return out; } CpuResource::CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor) - : Resource(std::move(name), ResourceType::CPU, device_id, enable_loader, enable_executor) {} + : Resource(std::move(name), ResourceType::CPU, device_id, enable_loader, enable_executor) { +} -void CpuResource::LoadFile(TaskPtr task) { +void +CpuResource::LoadFile(TaskPtr task) { task->Load(LoadType::DISK2CPU, 0); } -void CpuResource::Process(TaskPtr task) { +void +CpuResource::Process(TaskPtr task) { task->Execute(); } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/CpuResource.h b/cpp/src/scheduler/resource/CpuResource.h index 1f6f235621..2226523fdf 100644 --- a/cpp/src/scheduler/resource/CpuResource.h +++ b/cpp/src/scheduler/resource/CpuResource.h @@ -21,13 +21,12 @@ #include "Resource.h" - namespace zilliz { namespace milvus { namespace scheduler { class CpuResource : public Resource { -public: + public: explicit CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); @@ -38,7 +37,7 @@ public: friend std::ostream &operator<<(std::ostream &out, const CpuResource &resource); -protected: + protected: void LoadFile(TaskPtr task) override; @@ -46,6 +45,6 @@ protected: Process(TaskPtr task) override; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/DiskResource.cpp b/cpp/src/scheduler/resource/DiskResource.cpp index 6495396fdb..eee2424cc1 100644 --- a/cpp/src/scheduler/resource/DiskResource.cpp +++ b/cpp/src/scheduler/resource/DiskResource.cpp @@ -15,14 +15,17 @@ // specific language governing permissions and limitations // under the License. -#include "DiskResource.h" +#include "scheduler/resource/DiskResource.h" +#include +#include namespace zilliz { namespace milvus { namespace scheduler { -std::ostream &operator<<(std::ostream &out, const DiskResource &resource) { +std::ostream & +operator<<(std::ostream &out, const DiskResource &resource) { out << resource.Dump(); return out; } @@ -31,15 +34,14 @@ DiskResource::DiskResource(std::string name, uint64_t device_id, bool enable_loa : Resource(std::move(name), ResourceType::DISK, device_id, enable_loader, enable_executor) { } -void DiskResource::LoadFile(TaskPtr task) { - +void +DiskResource::LoadFile(TaskPtr task) { } -void DiskResource::Process(TaskPtr task) { - +void +DiskResource::Process(TaskPtr task) { } -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/DiskResource.h b/cpp/src/scheduler/resource/DiskResource.h index b9e7a4035c..a7caf5c662 100644 --- a/cpp/src/scheduler/resource/DiskResource.h +++ b/cpp/src/scheduler/resource/DiskResource.h @@ -17,16 +17,16 @@ #pragma once - #include "Resource.h" +#include namespace zilliz { namespace milvus { namespace scheduler { class DiskResource : public Resource { -public: + public: explicit DiskResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); @@ -37,7 +37,7 @@ public: friend std::ostream &operator<<(std::ostream &out, const DiskResource &resource); -protected: + protected: void LoadFile(TaskPtr task) override; @@ -45,6 +45,6 @@ protected: Process(TaskPtr task) override; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/GpuResource.cpp b/cpp/src/scheduler/resource/GpuResource.cpp index c3981e65cb..3c7abc0b29 100644 --- a/cpp/src/scheduler/resource/GpuResource.cpp +++ b/cpp/src/scheduler/resource/GpuResource.cpp @@ -16,29 +16,32 @@ // under the License. -#include "GpuResource.h" - +#include "scheduler/resource/GpuResource.h" namespace zilliz { namespace milvus { namespace scheduler { -std::ostream &operator<<(std::ostream &out, const GpuResource &resource) { +std::ostream & +operator<<(std::ostream &out, const GpuResource &resource) { out << resource.Dump(); return out; } GpuResource::GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor) - : Resource(std::move(name), ResourceType::GPU, device_id, enable_loader, enable_executor) {} + : Resource(std::move(name), ResourceType::GPU, device_id, enable_loader, enable_executor) { +} -void GpuResource::LoadFile(TaskPtr task) { +void +GpuResource::LoadFile(TaskPtr task) { task->Load(LoadType::CPU2GPU, device_id_); } -void GpuResource::Process(TaskPtr task) { +void +GpuResource::Process(TaskPtr task) { task->Execute(); } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/GpuResource.h b/cpp/src/scheduler/resource/GpuResource.h index f08e767c51..9f19b07464 100644 --- a/cpp/src/scheduler/resource/GpuResource.h +++ b/cpp/src/scheduler/resource/GpuResource.h @@ -17,16 +17,17 @@ #pragma once - #include "Resource.h" +#include +#include namespace zilliz { namespace milvus { namespace scheduler { class GpuResource : public Resource { -public: + public: explicit GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); @@ -37,7 +38,7 @@ public: friend std::ostream &operator<<(std::ostream &out, const GpuResource &resource); -protected: + protected: void LoadFile(TaskPtr task) override; @@ -45,6 +46,6 @@ protected: Process(TaskPtr task) override; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Node.cpp b/cpp/src/scheduler/resource/Node.cpp index 719bfb1427..cf652b8ba0 100644 --- a/cpp/src/scheduler/resource/Node.cpp +++ b/cpp/src/scheduler/resource/Node.cpp @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. +#include "scheduler/resource/Node.h" #include -#include "Node.h" - +#include namespace zilliz { namespace milvus { @@ -29,7 +29,8 @@ Node::Node() { id_ = counter++; } -std::vector Node::GetNeighbours() { +std::vector +Node::GetNeighbours() { std::lock_guard lk(mutex_); std::vector ret; for (auto &e : neighbours_) { @@ -38,7 +39,8 @@ std::vector Node::GetNeighbours() { return ret; } -std::string Node::Dump() { +std::string +Node::Dump() { std::stringstream ss; ss << "::neighbours:" << std::endl; for (auto &neighbour : neighbours_) { @@ -48,7 +50,8 @@ std::string Node::Dump() { return ss.str(); } -void Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connection) { +void +Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connection) { std::lock_guard lk(mutex_); if (auto s = neighbour_node.lock()) { neighbours_.emplace(std::make_pair(s->id_, Neighbour(neighbour_node, connection))); @@ -56,6 +59,6 @@ void Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &conn // else do nothing, consider it.. } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Node.h b/cpp/src/scheduler/resource/Node.h index 46506da10d..00337a7009 100644 --- a/cpp/src/scheduler/resource/Node.h +++ b/cpp/src/scheduler/resource/Node.h @@ -20,11 +20,11 @@ #include #include #include +#include -#include "../TaskTable.h" +#include "scheduler/TaskTable.h" #include "Connection.h" - namespace zilliz { namespace milvus { namespace scheduler { @@ -34,8 +34,9 @@ class Node; using NeighbourNodePtr = std::weak_ptr; struct Neighbour { - Neighbour(NeighbourNodePtr nei, Connection conn) - : neighbour_node(nei), connection(conn) {} + Neighbour(NeighbourNodePtr nei, Connection conn) + : neighbour_node(nei), connection(conn) { + } NeighbourNodePtr neighbour_node; Connection connection; @@ -43,7 +44,7 @@ struct Neighbour { // TODO(linxj): return type void -> Status class Node { -public: + public: Node(); void @@ -52,11 +53,11 @@ public: std::vector GetNeighbours(); -public: + public: std::string Dump(); -private: + private: std::mutex mutex_; uint8_t id_; std::map neighbours_; @@ -65,6 +66,6 @@ private: using NodePtr = std::shared_ptr; using NodeWPtr = std::weak_ptr; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Resource.cpp b/cpp/src/scheduler/resource/Resource.cpp index 22e7eb8502..4eb71b8ac5 100644 --- a/cpp/src/scheduler/resource/Resource.cpp +++ b/cpp/src/scheduler/resource/Resource.cpp @@ -15,10 +15,11 @@ // specific language governing permissions and limitations // under the License. -#include -#include "../Utils.h" -#include "Resource.h" +#include "scheduler/resource/Resource.h" +#include "scheduler/Utils.h" +#include +#include namespace zilliz { namespace milvus { @@ -100,7 +101,8 @@ Resource::NumOfTaskToExec() { return count; } -TaskTableItemPtr Resource::pick_task_load() { +TaskTableItemPtr +Resource::pick_task_load() { auto indexes = task_table_.PickToLoad(10); for (auto index : indexes) { // try to set one task loading, then return @@ -111,7 +113,8 @@ TaskTableItemPtr Resource::pick_task_load() { return nullptr; } -TaskTableItemPtr Resource::pick_task_execute() { +TaskTableItemPtr +Resource::pick_task_execute() { auto indexes = task_table_.PickToExecute(3); for (auto index : indexes) { // try to set one task executing, then return @@ -122,10 +125,13 @@ TaskTableItemPtr Resource::pick_task_execute() { return nullptr; } -void Resource::loader_function() { +void +Resource::loader_function() { while (running_) { std::unique_lock lock(load_mutex_); - load_cv_.wait(lock, [&] { return load_flag_; }); + load_cv_.wait(lock, [&] { + return load_flag_; + }); load_flag_ = false; lock.unlock(); while (true) { @@ -140,18 +146,20 @@ void Resource::loader_function() { subscriber_(std::static_pointer_cast(event)); } } - } } -void Resource::executor_function() { +void +Resource::executor_function() { if (subscriber_) { auto event = std::make_shared(shared_from_this()); subscriber_(std::static_pointer_cast(event)); } while (running_) { std::unique_lock lock(exec_mutex_); - exec_cv_.wait(lock, [&] { return exec_flag_; }); + exec_cv_.wait(lock, [&] { + return exec_flag_; + }); exec_flag_ = false; lock.unlock(); while (true) { @@ -172,10 +180,9 @@ void Resource::executor_function() { subscriber_(std::static_pointer_cast(event)); } } - } } -} -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Resource.h b/cpp/src/scheduler/resource/Resource.h index 0b0eaae304..1c18b1a2b2 100644 --- a/cpp/src/scheduler/resource/Resource.h +++ b/cpp/src/scheduler/resource/Resource.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -34,7 +35,6 @@ #include "Connection.h" #include "Node.h" - namespace zilliz { namespace milvus { namespace scheduler { @@ -104,7 +104,7 @@ class Resource : public Node, public std::enable_shared_from_this { return task_table_; } -public: + public: inline bool HasLoader() const { return enable_loader_; @@ -212,7 +212,6 @@ public: using ResourcePtr = std::shared_ptr; using ResourceWPtr = std::weak_ptr; -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/TestResource.cpp b/cpp/src/scheduler/resource/TestResource.cpp index b49ea12d77..25560cf7ee 100644 --- a/cpp/src/scheduler/resource/TestResource.cpp +++ b/cpp/src/scheduler/resource/TestResource.cpp @@ -15,14 +15,16 @@ // specific language governing permissions and limitations // under the License. -#include "TestResource.h" +#include "scheduler/resource/TestResource.h" +#include namespace zilliz { namespace milvus { namespace scheduler { -std::ostream &operator<<(std::ostream &out, const TestResource &resource) { +std::ostream & +operator<<(std::ostream &out, const TestResource &resource) { out << resource.Dump(); return out; } @@ -31,15 +33,16 @@ TestResource::TestResource(std::string name, uint64_t device_id, bool enable_loa : Resource(std::move(name), ResourceType::TEST, device_id, enable_loader, enable_executor) { } -void TestResource::LoadFile(TaskPtr task) { +void +TestResource::LoadFile(TaskPtr task) { task->Load(LoadType::TEST, 0); } -void TestResource::Process(TaskPtr task) { +void +TestResource::Process(TaskPtr task) { task->Execute(); } -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/TestResource.h b/cpp/src/scheduler/resource/TestResource.h index 861efbd825..ac83a42c60 100644 --- a/cpp/src/scheduler/resource/TestResource.h +++ b/cpp/src/scheduler/resource/TestResource.h @@ -17,16 +17,17 @@ #pragma once - #include "Resource.h" +#include +#include namespace zilliz { namespace milvus { namespace scheduler { class TestResource : public Resource { -public: + public: explicit TestResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); @@ -37,7 +38,7 @@ public: friend std::ostream &operator<<(std::ostream &out, const TestResource &resource); -protected: + protected: void LoadFile(TaskPtr task) override; @@ -45,6 +46,6 @@ protected: Process(TaskPtr task) override; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/DeleteTask.cpp b/cpp/src/scheduler/task/DeleteTask.cpp index 65a7a537fd..52579d67c6 100644 --- a/cpp/src/scheduler/task/DeleteTask.cpp +++ b/cpp/src/scheduler/task/DeleteTask.cpp @@ -16,19 +16,18 @@ // under the License. -#include "DeleteTask.h" - +#include "scheduler/task/DeleteTask.h" namespace zilliz { namespace milvus { namespace scheduler { XDeleteTask::XDeleteTask(const scheduler::DeleteJobPtr &delete_job) - : Task(TaskType::DeleteTask), delete_job_(delete_job) {} + : Task(TaskType::DeleteTask), delete_job_(delete_job) { +} void XDeleteTask::Load(LoadType type, uint8_t device_id) { - } void @@ -36,6 +35,6 @@ XDeleteTask::Execute() { delete_job_->ResourceDone(); } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/DeleteTask.h b/cpp/src/scheduler/task/DeleteTask.h index b64f9a977f..608960e7c8 100644 --- a/cpp/src/scheduler/task/DeleteTask.h +++ b/cpp/src/scheduler/task/DeleteTask.h @@ -20,15 +20,13 @@ #include "scheduler/job/DeleteJob.h" #include "Task.h" - namespace zilliz { namespace milvus { namespace scheduler { class XDeleteTask : public Task { -public: - explicit - XDeleteTask(const scheduler::DeleteJobPtr &job); + public: + explicit XDeleteTask(const scheduler::DeleteJobPtr &job); void Load(LoadType type, uint8_t device_id) override; @@ -36,10 +34,10 @@ public: void Execute() override; -public: + public: scheduler::DeleteJobPtr delete_job_; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/Path.h b/cpp/src/scheduler/task/Path.h index a463494b77..672dfff1b9 100644 --- a/cpp/src/scheduler/task/Path.h +++ b/cpp/src/scheduler/task/Path.h @@ -20,7 +20,6 @@ #include #include - namespace zilliz { namespace milvus { namespace scheduler { @@ -29,7 +28,8 @@ class Path { public: Path() = default; - Path(std::vector& path, uint64_t index) : path_(path), index_(index) {} + Path(std::vector &path, uint64_t index) : path_(path), index_(index) { + } void push_back(const std::string &str) { @@ -49,7 +49,6 @@ class Path { } else { return nullptr; } - } std::string @@ -67,14 +66,19 @@ class Path { return path_[index]; } - std::vector::iterator begin() { return path_.begin(); } - std::vector::iterator end() { return path_.end(); } + std::vector::iterator begin() { + return path_.begin(); + } + + std::vector::iterator end() { + return path_.end(); + } public: std::vector path_; uint64_t index_ = 0; }; -} -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/SearchTask.cpp b/cpp/src/scheduler/task/SearchTask.cpp index 284152f14e..0c205fcafa 100644 --- a/cpp/src/scheduler/task/SearchTask.cpp +++ b/cpp/src/scheduler/task/SearchTask.cpp @@ -15,15 +15,16 @@ // specific language governing permissions and limitations // under the License. -#include "SearchTask.h" +#include "scheduler/task/SearchTask.h" +#include "scheduler/job/SearchJob.h" #include "metrics/Metrics.h" #include "db/engine/EngineFactory.h" #include "utils/TimeRecorder.h" #include "utils/Log.h" #include -#include "scheduler/job/SearchJob.h" - +#include +#include namespace zilliz { namespace milvus { @@ -104,7 +105,6 @@ XSearchTask::XSearchTask(TableFileSchemaPtr file) (MetricType) file_->metric_type_, file_->nlist_); } - } void @@ -144,7 +144,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { s = Status(SERVER_UNEXPECTED_ERROR, error_msg); } - if (auto job = job_.lock()){ + if (auto job = job_.lock()) { auto search_job = std::static_pointer_cast(job); search_job->SearchDone(file_->id_); search_job->GetStatus() = s; @@ -183,7 +183,7 @@ XSearchTask::Execute() { server::CollectDurationMetrics metrics(index_type_); - std::vector output_ids; + std::vector output_ids; std::vector output_distance; if (auto job = job_.lock()) { @@ -192,7 +192,7 @@ XSearchTask::Execute() { uint64_t nq = search_job->nq(); uint64_t topk = search_job->topk(); uint64_t nprobe = search_job->nprobe(); - const float* vectors = search_job->vectors(); + const float *vectors = search_job->vectors(); output_ids.resize(topk * nq); output_distance.resize(topk * nq); @@ -236,11 +236,12 @@ XSearchTask::Execute() { index_engine_ = nullptr; } -Status XSearchTask::ClusterResult(const std::vector &output_ids, - const std::vector &output_distance, - uint64_t nq, - uint64_t topk, - scheduler::ResultSet &result_set) { +Status +XSearchTask::ClusterResult(const std::vector &output_ids, + const std::vector &output_distance, + uint64_t nq, + uint64_t topk, + scheduler::ResultSet &result_set) { if (output_ids.size() < nq * topk || output_distance.size() < nq * topk) { std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + " distance array size: " + std::to_string(output_distance.size()); @@ -275,10 +276,11 @@ Status XSearchTask::ClusterResult(const std::vector &output_ids, return Status::OK(); } -Status XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, - scheduler::Id2DistanceMap &distance_target, - uint64_t topk, - bool ascending) { +Status +XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, + scheduler::Id2DistanceMap &distance_target, + uint64_t topk, + bool ascending) { //Note: the score_src and score_target are already arranged by score in ascending order if (distance_src.empty()) { ENGINE_LOG_WARNING << "Empty distance source array"; @@ -349,10 +351,11 @@ Status XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, return Status::OK(); } -Status XSearchTask::TopkResult(scheduler::ResultSet &result_src, - uint64_t topk, - bool ascending, - scheduler::ResultSet &result_target) { +Status +XSearchTask::TopkResult(scheduler::ResultSet &result_src, + uint64_t topk, + bool ascending, + scheduler::ResultSet &result_target) { if (result_target.empty()) { result_target.swap(result_src); return Status::OK(); @@ -381,7 +384,6 @@ Status XSearchTask::TopkResult(scheduler::ResultSet &result_src, return Status::OK(); } - -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/SearchTask.h b/cpp/src/scheduler/task/SearchTask.h index 6f0ec6bc53..7c19ba20f9 100644 --- a/cpp/src/scheduler/task/SearchTask.h +++ b/cpp/src/scheduler/task/SearchTask.h @@ -21,6 +21,7 @@ #include "scheduler/job/SearchJob.h" #include "scheduler/Definition.h" +#include namespace zilliz { namespace milvus { @@ -28,9 +29,8 @@ namespace scheduler { // TODO: rewrite class XSearchTask : public Task { -public: - explicit - XSearchTask(TableFileSchemaPtr file); + public: + explicit XSearchTask(TableFileSchemaPtr file); void Load(LoadType type, uint8_t device_id) override; @@ -38,8 +38,8 @@ public: void Execute() override; -public: - static Status ClusterResult(const std::vector &output_ids, + public: + static Status ClusterResult(const std::vector &output_ids, const std::vector &output_distence, uint64_t nq, uint64_t topk, @@ -55,7 +55,7 @@ public: bool ascending, scheduler::ResultSet &result_target); -public: + public: TableFileSchemaPtr file_; size_t index_id_ = 0; @@ -66,6 +66,6 @@ public: static std::mutex merge_mutex_; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/Task.h b/cpp/src/scheduler/task/Task.h index 785ef9cfb7..3600c10f03 100644 --- a/cpp/src/scheduler/task/Task.h +++ b/cpp/src/scheduler/task/Task.h @@ -25,7 +25,6 @@ #include #include - namespace zilliz { namespace milvus { namespace scheduler { @@ -49,20 +48,22 @@ using TaskPtr = std::shared_ptr; // TODO: re-design class Task { -public: - explicit - Task(TaskType type) : type_(type) {} + public: + explicit Task(TaskType type) : type_(type) { + } /* * Just Getter; */ inline TaskType - Type() const { return type_; } + Type() const { + return type_; + } /* * Transport path; */ - inline Path& + inline Path & path() { return task_path_; } @@ -75,14 +76,14 @@ public: return label_; } -public: + public: virtual void Load(LoadType type, uint8_t device_id) = 0; virtual void Execute() = 0; -public: + public: Path task_path_; // std::vector search_contexts_; scheduler::JobWPtr job_; @@ -90,7 +91,6 @@ public: TaskLabelPtr label_ = nullptr; }; - -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/TestTask.cpp b/cpp/src/scheduler/task/TestTask.cpp index 32e0a8587e..fc66e56269 100644 --- a/cpp/src/scheduler/task/TestTask.cpp +++ b/cpp/src/scheduler/task/TestTask.cpp @@ -15,16 +15,15 @@ // specific language governing permissions and limitations // under the License. - -#include -#include "TestTask.h" - +#include "scheduler/task/TestTask.h" +#include "cache/GpuCacheMgr.h" namespace zilliz { namespace milvus { namespace scheduler { -TestTask::TestTask(TableFileSchemaPtr &file) : XSearchTask(file) {} +TestTask::TestTask(TableFileSchemaPtr &file) : XSearchTask(file) { +} void TestTask::Load(LoadType type, uint8_t device_id) { @@ -44,10 +43,11 @@ TestTask::Execute() { void TestTask::Wait() { std::unique_lock lock(mutex_); - cv_.wait(lock, [&] { return done_; }); -} - -} -} + cv_.wait(lock, [&] { + return done_; + }); } +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/TestTask.h b/cpp/src/scheduler/task/TestTask.h index 3367506d88..7051080861 100644 --- a/cpp/src/scheduler/task/TestTask.h +++ b/cpp/src/scheduler/task/TestTask.h @@ -19,17 +19,15 @@ #include "SearchTask.h" - namespace zilliz { namespace milvus { namespace scheduler { class TestTask : public XSearchTask { -public: - explicit - TestTask(TableFileSchemaPtr& file); + public: + explicit TestTask(TableFileSchemaPtr &file); -public: + public: void Load(LoadType type, uint8_t device_id) override; @@ -39,7 +37,7 @@ public: void Wait(); -public: + public: uint64_t load_count_ = 0; uint64_t exec_count_ = 0; @@ -48,7 +46,6 @@ public: std::condition_variable cv_; }; - -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/tasklabel/BroadcastLabel.h b/cpp/src/scheduler/tasklabel/BroadcastLabel.h index 1c89460df0..6fca107864 100644 --- a/cpp/src/scheduler/tasklabel/BroadcastLabel.h +++ b/cpp/src/scheduler/tasklabel/BroadcastLabel.h @@ -21,19 +21,19 @@ #include - namespace zilliz { namespace milvus { namespace scheduler { - class BroadcastLabel : public TaskLabel { -public: - BroadcastLabel() : TaskLabel(TaskLabelType::BROADCAST) {} + public: + BroadcastLabel() : TaskLabel(TaskLabelType::BROADCAST) { + } }; using BroadcastLabelPtr = std::shared_ptr; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz + diff --git a/cpp/src/scheduler/tasklabel/DefaultLabel.h b/cpp/src/scheduler/tasklabel/DefaultLabel.h index d61028117d..7943c4f7c1 100644 --- a/cpp/src/scheduler/tasklabel/DefaultLabel.h +++ b/cpp/src/scheduler/tasklabel/DefaultLabel.h @@ -21,20 +21,18 @@ #include - namespace zilliz { namespace milvus { namespace scheduler { class DefaultLabel : public TaskLabel { -public: - DefaultLabel() : TaskLabel(TaskLabelType::DEFAULT) {} + public: + DefaultLabel() : TaskLabel(TaskLabelType::DEFAULT) { + } }; using DefaultLabelPtr = std::shared_ptr; -} -} -} - - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/tasklabel/SpecResLabel.h b/cpp/src/scheduler/tasklabel/SpecResLabel.h index 8e94a7677b..cfc5aa9400 100644 --- a/cpp/src/scheduler/tasklabel/SpecResLabel.h +++ b/cpp/src/scheduler/tasklabel/SpecResLabel.h @@ -22,7 +22,6 @@ #include #include - class Resource; using ResourceWPtr = std::weak_ptr; @@ -32,9 +31,10 @@ namespace milvus { namespace scheduler { class SpecResLabel : public TaskLabel { -public: - SpecResLabel(const ResourceWPtr &resource) - : TaskLabel(TaskLabelType::SPECIFIED_RESOURCE), resource_(resource) {} + public: + explicit SpecResLabel(const ResourceWPtr &resource) + : TaskLabel(TaskLabelType::SPECIFIED_RESOURCE), resource_(resource) { + } inline ResourceWPtr & resource() { @@ -46,14 +46,13 @@ public: return resource_name_; } -private: + private: ResourceWPtr resource_; std::string resource_name_; }; using SpecResLabelPtr = std::shared_ptr(); -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/tasklabel/TaskLabel.h b/cpp/src/scheduler/tasklabel/TaskLabel.h index 5d43adb7db..d8d404e094 100644 --- a/cpp/src/scheduler/tasklabel/TaskLabel.h +++ b/cpp/src/scheduler/tasklabel/TaskLabel.h @@ -30,23 +30,22 @@ enum class TaskLabelType { }; class TaskLabel { -public: + public: inline TaskLabelType Type() const { return type_; } -protected: - explicit - TaskLabel(TaskLabelType type) : type_(type) {} + protected: + explicit TaskLabel(TaskLabelType type) : type_(type) { + } -private: + private: TaskLabelType type_; }; using TaskLabelPtr = std::shared_ptr; -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/sdk/examples/grpcsimple/main.cpp b/cpp/src/sdk/examples/grpcsimple/main.cpp index 4de060590c..166707259a 100644 --- a/cpp/src/sdk/examples/grpcsimple/main.cpp +++ b/cpp/src/sdk/examples/grpcsimple/main.cpp @@ -23,8 +23,8 @@ #include "src/ClientTest.h" -void print_help(const std::string &app_name); - +void +print_help(const std::string &app_name); int main(int argc, char *argv[]) { @@ -56,8 +56,7 @@ main(int argc, char *argv[]) { break; } case 'h': - default: - print_help(app_name); + default:print_help(app_name); return EXIT_SUCCESS; } } @@ -77,4 +76,4 @@ print_help(const std::string &app_name) { printf(" -p --port Server port, default 19530\n"); printf(" -h --help Print help information\n"); printf("\n"); -} \ No newline at end of file +} diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp index 16fc2ee168..e29bcc30f3 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "ClientTest.h" +#include "sdk/examples/grpcsimple/src/ClientTest.h" #include "MilvusApi.h" #include "cache/CpuCacheMgr.h" @@ -24,15 +24,17 @@ #include #include #include - -using namespace milvus; +#include +#include +#include //#define SET_VECTOR_IDS; namespace { -std::string GetTableName(); +const std::string& +GetTableName(); -const std::string TABLE_NAME = GetTableName(); +const char* TABLE_NAME = GetTableName().c_str(); constexpr int64_t TABLE_DIMENSION = 512; constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; constexpr int64_t BATCH_ROW_COUNT = 100000; @@ -44,26 +46,28 @@ constexpr int64_t SECONDS_EACH_HOUR = 3600; #define BLOCK_SPLITER std::cout << "===========================================" << std::endl; -void PrintTableSchema(const TableSchema& tb_schema) { +void +PrintTableSchema(const milvus::TableSchema &tb_schema) { BLOCK_SPLITER std::cout << "Table name: " << tb_schema.table_name << std::endl; std::cout << "Table dimension: " << tb_schema.dimension << std::endl; BLOCK_SPLITER } -void PrintSearchResult(const std::vector>& search_record_array, - const std::vector& topk_query_result_array) { +void +PrintSearchResult(const std::vector> &search_record_array, + const std::vector &topk_query_result_array) { BLOCK_SPLITER std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl; int32_t index = 0; - for(auto& result : topk_query_result_array) { + for (auto &result : topk_query_result_array) { auto search_id = search_record_array[index].first; index++; std::cout << "No." << std::to_string(index) << " vector " << std::to_string(search_id) << " top " << std::to_string(result.query_result_arrays.size()) << " search result:" << std::endl; - for(auto& item : result.query_result_arrays) { + for (auto &item : result.query_result_arrays) { std::cout << "\t" << std::to_string(item.id) << "\tdistance:" << std::to_string(item.distance); std::cout << std::endl; } @@ -72,80 +76,88 @@ void PrintSearchResult(const std::vector>& search_ BLOCK_SPLITER } -std::string CurrentTime() { +std::string +CurrentTime() { time_t tt; - time( &tt ); - tt = tt + 8*SECONDS_EACH_HOUR; - tm* t= gmtime( &tt ); + time(&tt); + tt = tt + 8 * SECONDS_EACH_HOUR; + tm t; + gmtime_r(&tt, &t); - std::string str = std::to_string(t->tm_year + 1900) + "_" + std::to_string(t->tm_mon + 1) - + "_" + std::to_string(t->tm_mday) + "_" + std::to_string(t->tm_hour) - + "_" + std::to_string(t->tm_min) + "_" + std::to_string(t->tm_sec); + std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) + + "_" + std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) + + "_" + std::to_string(t.tm_min) + "_" + std::to_string(t.tm_sec); return str; } -std::string CurrentTmDate(int64_t offset_day = 0) { +std::string +CurrentTmDate(int64_t offset_day = 0) { time_t tt; - time( &tt ); - tt = tt + 8*SECONDS_EACH_HOUR; - tt = tt + 24*SECONDS_EACH_HOUR*offset_day; - tm* t= gmtime( &tt ); + time(&tt); + tt = tt + 8 * SECONDS_EACH_HOUR; + tt = tt + 24 * SECONDS_EACH_HOUR * offset_day; + tm t; + gmtime_r(&tt, &t); - std::string str = std::to_string(t->tm_year + 1900) + "-" + std::to_string(t->tm_mon + 1) - + "-" + std::to_string(t->tm_mday); + std::string str = std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1) + + "-" + std::to_string(t.tm_mday); return str; } -std::string GetTableName() { - static std::string s_id(CurrentTime()); - return "tbl_" + s_id; +const std::string& +GetTableName() { + static std::string s_id("tbl_" + CurrentTime()); + return s_id; } -TableSchema BuildTableSchema() { - TableSchema tb_schema; +milvus::TableSchema +BuildTableSchema() { + milvus::TableSchema tb_schema; tb_schema.table_name = TABLE_NAME; tb_schema.dimension = TABLE_DIMENSION; tb_schema.index_file_size = TABLE_INDEX_FILE_SIZE; - tb_schema.metric_type = MetricType::L2; + tb_schema.metric_type = milvus::MetricType::L2; return tb_schema; } -void BuildVectors(int64_t from, int64_t to, - std::vector& vector_record_array) { - if(to <= from){ +void +BuildVectors(int64_t from, int64_t to, + std::vector &vector_record_array) { + if (to <= from) { return; } vector_record_array.clear(); for (int64_t k = from; k < to; k++) { - RowRecord record; + milvus::RowRecord record; record.data.resize(TABLE_DIMENSION); - for(int64_t i = 0; i < TABLE_DIMENSION; i++) { - record.data[i] = (float)(k%(i+1)); + for (int64_t i = 0; i < TABLE_DIMENSION; i++) { + record.data[i] = (float) (k % (i + 1)); } vector_record_array.emplace_back(record); } } -void Sleep(int seconds) { +void +Sleep(int seconds) { std::cout << "Waiting " << seconds << " seconds ..." << std::endl; sleep(seconds); } class TimeRecorder { public: - explicit TimeRecorder(const std::string& title) + explicit TimeRecorder(const std::string &title) : title_(title) { start_ = std::chrono::system_clock::now(); } ~TimeRecorder() { std::chrono::system_clock::time_point end = std::chrono::system_clock::now(); - long span = (std::chrono::duration_cast (end - start_)).count(); + int64_t span = (std::chrono::duration_cast(end - start_)).count(); std::cout << title_ << " totally cost: " << span << " ms" << std::endl; } @@ -154,14 +166,15 @@ class TimeRecorder { std::chrono::system_clock::time_point start_; }; -void CheckResult(const std::vector>& search_record_array, - const std::vector& topk_query_result_array) { +void +CheckResult(const std::vector> &search_record_array, + const std::vector &topk_query_result_array) { BLOCK_SPLITER int64_t index = 0; - for(auto& result : topk_query_result_array) { + for (auto &result : topk_query_result_array) { auto result_id = result.query_result_arrays[0].id; auto search_id = search_record_array[index++].first; - if(result_id != search_id) { + if (result_id != search_id) { std::cout << "The top 1 result is wrong: " << result_id << " vs. " << search_id << std::endl; } else { @@ -171,42 +184,45 @@ void CheckResult(const std::vector>& search_record BLOCK_SPLITER } -void DoSearch(std::shared_ptr conn, - const std::vector>& search_record_array, - const std::string& phase_name) { - std::vector query_range_array; - Range rg; +void +DoSearch(std::shared_ptr conn, + const std::vector> &search_record_array, + const std::string &phase_name) { + std::vector query_range_array; + milvus::Range rg; rg.start_value = CurrentTmDate(); rg.end_value = CurrentTmDate(1); query_range_array.emplace_back(rg); - std::vector record_array; - for(auto& pair : search_record_array) { + std::vector record_array; + for (auto &pair : search_record_array) { record_array.push_back(pair.second); } auto start = std::chrono::high_resolution_clock::now(); - std::vector topk_query_result_array; + std::vector topk_query_result_array; { TimeRecorder rc(phase_name); - Status stat = conn->Search(TABLE_NAME, record_array, query_range_array, TOP_K, 32, topk_query_result_array); + milvus::Status stat = + conn->Search(TABLE_NAME, record_array, query_range_array, TOP_K, 32, topk_query_result_array); std::cout << "SearchVector function call status: " << stat.message() << std::endl; } auto finish = std::chrono::high_resolution_clock::now(); - std::cout << "SEARCHVECTOR COST: " << std::chrono::duration_cast>(finish - start).count() << "s\n"; + std::cout << "SEARCHVECTOR COST: " + << std::chrono::duration_cast>(finish - start).count() << "s\n"; PrintSearchResult(search_record_array, topk_query_result_array); CheckResult(search_record_array, topk_query_result_array); } -} +} // namespace void -ClientTest::Test(const std::string& address, const std::string& port) { - std::shared_ptr conn = Connection::Create(); +ClientTest::Test(const std::string &address, const std::string &port) { + std::shared_ptr conn = milvus::Connection::Create(); {//connect server - ConnectParam param = {address, port}; - Status stat = conn->Connect(param); + milvus::ConnectParam param = {address, port}; + milvus::Status stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } @@ -222,10 +238,10 @@ ClientTest::Test(const std::string& address, const std::string& port) { { std::vector tables; - Status stat = conn->ShowTables(tables); + milvus::Status stat = conn->ShowTables(tables); std::cout << "ShowTables function call status: " << stat.message() << std::endl; std::cout << "All tables: " << std::endl; - for(auto& table : tables) { + for (auto &table : tables) { int64_t row_count = 0; // conn->DropTable(table); stat = conn->CountTable(table, row_count); @@ -234,28 +250,28 @@ ClientTest::Test(const std::string& address, const std::string& port) { } {//create table - TableSchema tb_schema = BuildTableSchema(); - Status stat = conn->CreateTable(tb_schema); + milvus::TableSchema tb_schema = BuildTableSchema(); + milvus::Status stat = conn->CreateTable(tb_schema); std::cout << "CreateTable function call status: " << stat.message() << std::endl; PrintTableSchema(tb_schema); bool has_table = conn->HasTable(tb_schema.table_name); - if(has_table) { + if (has_table) { std::cout << "Table is created" << std::endl; } } {//describe table - TableSchema tb_schema; - Status stat = conn->DescribeTable(TABLE_NAME, tb_schema); + milvus::TableSchema tb_schema; + milvus::Status stat = conn->DescribeTable(TABLE_NAME, tb_schema); std::cout << "DescribeTable function call status: " << stat.message() << std::endl; PrintTableSchema(tb_schema); } - std::vector> search_record_array; + std::vector> search_record_array; {//insert vectors for (int i = 0; i < ADD_VECTOR_LOOP; i++) {//add vectors - std::vector record_array; + std::vector record_array; int64_t begin_index = i * BATCH_ROW_COUNT; BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array); @@ -268,21 +284,21 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::vector record_ids; //generate user defined ids - for(int k = 0; k < BATCH_ROW_COUNT; k++) { - record_ids.push_back(i*BATCH_ROW_COUNT+k); + for (int k = 0; k < BATCH_ROW_COUNT; k++) { + record_ids.push_back(i * BATCH_ROW_COUNT + k); } auto start = std::chrono::high_resolution_clock::now(); - Status stat = conn->Insert(TABLE_NAME, record_array, record_ids); + milvus::Status stat = conn->Insert(TABLE_NAME, record_array, record_ids); auto finish = std::chrono::high_resolution_clock::now(); - std::cout << "InsertVector cost: " << std::chrono::duration_cast>(finish - start).count() << "s\n"; - + std::cout << "InsertVector cost: " + << std::chrono::duration_cast>(finish - start).count() << "s\n"; std::cout << "InsertVector function call status: " << stat.message() << std::endl; std::cout << "Returned id array count: " << record_ids.size() << std::endl; - if(search_record_array.size() < NQ) { + if (search_record_array.size() < NQ) { search_record_array.push_back( std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET])); } @@ -293,27 +309,27 @@ ClientTest::Test(const std::string& address, const std::string& port) { Sleep(2); int64_t row_count = 0; - Status stat = conn->CountTable(TABLE_NAME, row_count); + milvus::Status stat = conn->CountTable(TABLE_NAME, row_count); std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; // DoSearch(conn, search_record_array, "Search without index"); } {//wait unit build index finish std::cout << "Wait until create all index done" << std::endl; - IndexParam index; + milvus::IndexParam index; index.table_name = TABLE_NAME; - index.index_type = IndexType::gpu_ivfsq8; + index.index_type = milvus::IndexType::gpu_ivfsq8; index.nlist = 16384; - Status stat = conn->CreateIndex(index); + milvus::Status stat = conn->CreateIndex(index); std::cout << "CreateIndex function call status: " << stat.message() << std::endl; - IndexParam index2; + milvus::IndexParam index2; stat = conn->DescribeIndex(TABLE_NAME, index2); std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; } {//preload table - Status stat = conn->PreloadTable(TABLE_NAME); + milvus::Status stat = conn->PreloadTable(TABLE_NAME); std::cout << "PreloadTable function call status: " << stat.message() << std::endl; } @@ -325,7 +341,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { } {//delete index - Status stat = conn->DropIndex(TABLE_NAME); + milvus::Status stat = conn->DropIndex(TABLE_NAME); std::cout << "DropIndex function call status: " << stat.message() << std::endl; int64_t row_count = 0; @@ -334,11 +350,11 @@ ClientTest::Test(const std::string& address, const std::string& port) { } {//delete by range - Range rg; + milvus::Range rg; rg.start_value = CurrentTmDate(-2); rg.end_value = CurrentTmDate(-3); - Status stat = conn->DeleteByRange(rg, TABLE_NAME); + milvus::Status stat = conn->DeleteByRange(rg, TABLE_NAME); std::cout << "DeleteByRange function call status: " << stat.message() << std::endl; } @@ -351,7 +367,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::string status = conn->ServerStatus(); std::cout << "Server status before disconnect: " << status << std::endl; } - Connection::Destroy(conn); + milvus::Connection::Destroy(conn); {//server status std::string status = conn->ServerStatus(); std::cout << "Server status after disconnect: " << status << std::endl; diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h index d95897927e..165c36a180 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h @@ -20,6 +20,6 @@ #include class ClientTest { -public: - void Test(const std::string& address, const std::string& port); -}; \ No newline at end of file + public: + void Test(const std::string &address, const std::string &port); +}; diff --git a/cpp/src/sdk/grpc/ClientProxy.cpp b/cpp/src/sdk/grpc/ClientProxy.cpp index 3bcfd699fe..f4cd1e9a38 100644 --- a/cpp/src/sdk/grpc/ClientProxy.cpp +++ b/cpp/src/sdk/grpc/ClientProxy.cpp @@ -15,16 +15,20 @@ // specific language governing permissions and limitations // under the License. -#include "ClientProxy.h" -#include "version.h" -#include "milvus.grpc.pb.h" +#include "sdk/grpc/ClientProxy.h" +#include "../../../version.h" +#include "grpc/gen-milvus/milvus.grpc.pb.h" + +#include +#include +#include + //#define GRPC_MULTIPLE_THREAD; namespace milvus { - bool UriCheck(const std::string &uri) { - size_t index = uri.find_first_of(':', 0); + size_t index = uri.find_first_of(':', 0); if (index == std::string::npos) { return false; } else { @@ -79,7 +83,7 @@ ClientProxy::Disconnect() { connected_ = false; channel_.reset(); return status; - }catch (std::exception &ex) { + } catch (std::exception &ex) { return Status(StatusCode::UnknownError, "failed to disconnect: " + std::string(ex.what())); } } @@ -96,7 +100,7 @@ ClientProxy::CreateTable(const TableSchema ¶m) { schema.set_table_name(param.table_name); schema.set_dimension(param.dimension); schema.set_index_file_size(param.index_file_size); - schema.set_metric_type((int32_t)param.metric_type); + schema.set_metric_type((int32_t) param.metric_type); return client_ptr_->CreateTable(schema); } catch (std::exception &ex) { @@ -127,13 +131,12 @@ ClientProxy::DropTable(const std::string &table_name) { Status ClientProxy::CreateIndex(const IndexParam &index_param) { try { - //TODO:add index params + //TODO: add index params ::milvus::grpc::IndexParam grpc_index_param; grpc_index_param.set_table_name(index_param.table_name); - grpc_index_param.mutable_index()->set_index_type((int32_t)index_param.index_type); + grpc_index_param.mutable_index()->set_index_type((int32_t) index_param.index_type); grpc_index_param.mutable_index()->set_nlist(index_param.nlist); return client_ptr_->CreateIndex(grpc_index_param); - } catch (std::exception &ex) { return Status(StatusCode::UnknownError, "failed to build index: " + std::string(ex.what())); } @@ -141,8 +144,8 @@ ClientProxy::CreateIndex(const IndexParam &index_param) { Status ClientProxy::Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) { + const std::vector &record_array, + std::vector &id_array) { Status status = Status::OK(); try { //////////////////////////////////////////////////////////////////////////// @@ -181,7 +184,9 @@ ClientProxy::Insert(const std::string &table_name, } std::for_each(threads.begin(), threads.end(), std::mem_fn(&std::thread::join)); auto finish = std::chrono::high_resolution_clock::now(); - std::cout << "InsertVector cost: " << std::chrono::duration_cast>(finish - start).count() << "s\n"; + std::cout << + "InsertVector cost: " << std::chrono::duration_cast>(finish - start).count() + << "s\n"; std::cout << "*****************************************************\n"; for (size_t i = 0; i < thread_count; i++) { @@ -213,9 +218,7 @@ ClientProxy::Insert(const std::string &table_name, id_array.push_back(vector_ids.vector_id_array(i)); } } - #endif - } catch (std::exception &ex) { return Status(StatusCode::UnknownError, "fail to add vector: " + std::string(ex.what())); } @@ -225,11 +228,11 @@ ClientProxy::Insert(const std::string &table_name, Status ClientProxy::Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) { + const std::vector &query_record_array, + const std::vector &query_range_array, + int64_t topk, + int64_t nprobe, + std::vector &topk_query_result_array) { try { //step 1: convert vectors data ::milvus::grpc::SearchParam search_param; @@ -267,11 +270,9 @@ ClientProxy::Search(const std::string &table_name, topk_query_result_array.emplace_back(result); } return status; - } catch (std::exception &ex) { return Status(StatusCode::UnknownError, "fail to search vectors: " + std::string(ex.what())); } - } Status @@ -284,13 +285,12 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch table_schema.table_name = grpc_schema.table_name(); table_schema.dimension = grpc_schema.dimension(); table_schema.index_file_size = grpc_schema.index_file_size(); - table_schema.metric_type = (MetricType)grpc_schema.metric_type(); + table_schema.metric_type = (MetricType) grpc_schema.metric_type(); return status; } catch (std::exception &ex) { return Status(StatusCode::UnknownError, "fail to describe table: " + std::string(ex.what())); } - } Status @@ -316,7 +316,6 @@ ClientProxy::ShowTables(std::vector &table_array) { table_array[i] = table_name_list.table_names(i); } return status; - } catch (std::exception &ex) { return Status(StatusCode::UnknownError, "fail to show tables: " + std::string(ex.what())); } @@ -396,11 +395,10 @@ ClientProxy::DescribeIndex(const std::string &table_name, IndexParam &index_para grpc_table_name.set_table_name(table_name); ::milvus::grpc::IndexParam grpc_index_param; Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param); - index_param.index_type = (IndexType)(grpc_index_param.mutable_index()->index_type()); + index_param.index_type = (IndexType) (grpc_index_param.mutable_index()->index_type()); index_param.nlist = grpc_index_param.mutable_index()->nlist(); return status; - } catch (std::exception &ex) { return Status(StatusCode::UnknownError, "fail to describe index: " + std::string(ex.what())); } @@ -418,4 +416,4 @@ ClientProxy::DropIndex(const std::string &table_name) const { } } -} +} // namespace milvus diff --git a/cpp/src/sdk/grpc/ClientProxy.h b/cpp/src/sdk/grpc/ClientProxy.h index c3f79d7209..044ce44847 100644 --- a/cpp/src/sdk/grpc/ClientProxy.h +++ b/cpp/src/sdk/grpc/ClientProxy.h @@ -20,88 +20,92 @@ #include "MilvusApi.h" #include "GrpcClient.h" +#include +#include +#include + namespace milvus { class ClientProxy : public Connection { -public: + public: // Implementations of the Connection interface - virtual Status + Status Connect(const ConnectParam ¶m) override; - virtual Status + Status Connect(const std::string &uri) override; - virtual Status + Status Connected() const override; - virtual Status + Status Disconnect() override; - virtual Status + Status CreateTable(const TableSchema ¶m) override; - virtual bool + bool HasTable(const std::string &table_name) override; - virtual Status + Status DropTable(const std::string &table_name) override; - virtual Status + Status CreateIndex(const IndexParam &index_param) override; - virtual Status + Status Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) override; + const std::vector &record_array, + std::vector &id_array) override; - virtual Status + Status Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) override; + const std::vector &query_record_array, + const std::vector &query_range_array, + int64_t topk, + int64_t nprobe, + std::vector &topk_query_result_array) override; - virtual Status + Status DescribeTable(const std::string &table_name, TableSchema &table_schema) override; - virtual Status + Status CountTable(const std::string &table_name, int64_t &row_count) override; - virtual Status + Status ShowTables(std::vector &table_array) override; - virtual std::string + std::string ClientVersion() const override; - virtual std::string + std::string ServerVersion() const override; - virtual std::string + std::string ServerStatus() const override; - virtual std::string + std::string DumpTaskTables() const override; - virtual Status + Status DeleteByRange(Range &range, const std::string &table_name) override; - virtual Status + Status PreloadTable(const std::string &table_name) const override; - virtual Status + Status DescribeIndex(const std::string &table_name, IndexParam &index_param) const override; - virtual Status + Status DropIndex(const std::string &table_name) const override; -private: + private: std::shared_ptr<::grpc::Channel> channel_; -private: + private: std::shared_ptr client_ptr_; bool connected_ = false; }; -} +} // namespace milvus diff --git a/cpp/src/sdk/grpc/GrpcClient.cpp b/cpp/src/sdk/grpc/GrpcClient.cpp index 67bb85da8e..df86f0a3e7 100644 --- a/cpp/src/sdk/grpc/GrpcClient.cpp +++ b/cpp/src/sdk/grpc/GrpcClient.cpp @@ -15,13 +15,17 @@ // specific language governing permissions and limitations // under the License. +#include "sdk/grpc/GrpcClient.h" + #include #include #include #include #include -#include "GrpcClient.h" +#include +#include +#include using grpc::Channel; using grpc::ClientContext; @@ -31,15 +35,14 @@ using grpc::ClientWriter; using grpc::Status; namespace milvus { -GrpcClient::GrpcClient(std::shared_ptr<::grpc::Channel>& channel) - : stub_(::milvus::grpc::MilvusService::NewStub(channel)) { - +GrpcClient::GrpcClient(std::shared_ptr<::grpc::Channel> &channel) + : stub_(::milvus::grpc::MilvusService::NewStub(channel)) { } GrpcClient::~GrpcClient() = default; Status -GrpcClient::CreateTable(const ::milvus::grpc::TableSchema& table_schema) { +GrpcClient::CreateTable(const ::milvus::grpc::TableSchema &table_schema) { ClientContext context; grpc::Status response; ::grpc::Status grpc_status = stub_->CreateTable(&context, table_schema, &response); @@ -57,8 +60,8 @@ GrpcClient::CreateTable(const ::milvus::grpc::TableSchema& table_schema) { } bool -GrpcClient::HasTable(const ::milvus::grpc::TableName& table_name, - Status& status) { +GrpcClient::HasTable(const ::milvus::grpc::TableName &table_name, + Status &status) { ClientContext context; ::milvus::grpc::BoolReply response; ::grpc::Status grpc_status = stub_->HasTable(&context, table_name, &response); @@ -76,7 +79,7 @@ GrpcClient::HasTable(const ::milvus::grpc::TableName& table_name, } Status -GrpcClient::DropTable(const ::milvus::grpc::TableName& table_name) { +GrpcClient::DropTable(const ::milvus::grpc::TableName &table_name) { ClientContext context; grpc::Status response; ::grpc::Status grpc_status = stub_->DropTable(&context, table_name, &response); @@ -94,7 +97,7 @@ GrpcClient::DropTable(const ::milvus::grpc::TableName& table_name) { } Status -GrpcClient::CreateIndex(const ::milvus::grpc::IndexParam& index_param) { +GrpcClient::CreateIndex(const ::milvus::grpc::IndexParam &index_param) { ClientContext context; grpc::Status response; ::grpc::Status grpc_status = stub_->CreateIndex(&context, index_param, &response); @@ -112,9 +115,9 @@ GrpcClient::CreateIndex(const ::milvus::grpc::IndexParam& index_param) { } void -GrpcClient::Insert(::milvus::grpc::VectorIds& vector_ids, - const ::milvus::grpc::InsertParam& insert_param, - Status& status) { +GrpcClient::Insert(::milvus::grpc::VectorIds &vector_ids, + const ::milvus::grpc::InsertParam &insert_param, + Status &status) { ClientContext context; ::grpc::Status grpc_status = stub_->Insert(&context, insert_param, &vector_ids); @@ -133,7 +136,7 @@ GrpcClient::Insert(::milvus::grpc::VectorIds& vector_ids, } Status -GrpcClient::Search(::milvus::grpc::TopKQueryResultList& topk_query_result_list, +GrpcClient::Search(::milvus::grpc::TopKQueryResultList &topk_query_result_list, const ::milvus::grpc::SearchParam &search_param) { ::milvus::grpc::TopKQueryResult query_result; ClientContext context; @@ -154,8 +157,8 @@ GrpcClient::Search(::milvus::grpc::TopKQueryResultList& topk_query_result_list, } Status -GrpcClient::DescribeTable(::milvus::grpc::TableSchema& grpc_schema, - const std::string& table_name) { +GrpcClient::DescribeTable(::milvus::grpc::TableSchema &grpc_schema, + const std::string &table_name) { ClientContext context; ::milvus::grpc::TableName grpc_tablename; grpc_tablename.set_table_name(table_name); @@ -170,14 +173,14 @@ GrpcClient::DescribeTable(::milvus::grpc::TableSchema& grpc_schema, if (grpc_schema.status().error_code() != grpc::SUCCESS) { std::cerr << grpc_schema.status().reason() << std::endl; return Status(StatusCode::ServerFailed, - grpc_schema.status().reason()); + grpc_schema.status().reason()); } return Status::OK(); } int64_t -GrpcClient::CountTable(const std::string& table_name, Status& status) { +GrpcClient::CountTable(const std::string &table_name, Status &status) { ClientContext context; ::milvus::grpc::TableRowCount response; ::milvus::grpc::TableName grpc_tablename; @@ -186,7 +189,7 @@ GrpcClient::CountTable(const std::string& table_name, Status& status) { if (!grpc_status.ok()) { std::cerr << "DescribeTable rpc failed!" << std::endl; - status = Status(StatusCode::RPCFailed, grpc_status.error_message()); + status = Status(StatusCode::RPCFailed, grpc_status.error_message()); return -1; } @@ -223,7 +226,7 @@ GrpcClient::ShowTables(milvus::grpc::TableNameList &table_name_list) { Status GrpcClient::Cmd(std::string &result, - const std::string& cmd) { + const std::string &cmd) { ClientContext context; ::milvus::grpc::StringReply response; ::milvus::grpc::Command command; @@ -321,4 +324,4 @@ GrpcClient::DropIndex(grpc::TableName &table_name) { return Status::OK(); } -} \ No newline at end of file +} // namespace milvus diff --git a/cpp/src/sdk/grpc/GrpcClient.h b/cpp/src/sdk/grpc/GrpcClient.h index 08714d10e0..8f81e83ae8 100644 --- a/cpp/src/sdk/grpc/GrpcClient.h +++ b/cpp/src/sdk/grpc/GrpcClient.h @@ -16,6 +16,11 @@ // under the License. #pragma once + +#include "MilvusApi.h" +#include "grpc/gen-milvus/milvus.grpc.pb.h" +//#include "grpc/gen-status/status.grpc.pb.h" + #include #include #include @@ -28,55 +33,48 @@ #include #include #include -#include "MilvusApi.h" - -#include "milvus.grpc.pb.h" -//#include "status.grpc.pb.h" - -#include namespace milvus { class GrpcClient { -public: - explicit - GrpcClient(std::shared_ptr<::grpc::Channel>& channel); + public: + explicit GrpcClient(std::shared_ptr<::grpc::Channel> &channel); virtual ~GrpcClient(); Status - CreateTable(const grpc::TableSchema& table_schema); + CreateTable(const grpc::TableSchema &table_schema); bool - HasTable(const grpc::TableName& table_name, Status& status); + HasTable(const grpc::TableName &table_name, Status &status); Status - DropTable(const grpc::TableName& table_name); + DropTable(const grpc::TableName &table_name); Status - CreateIndex(const grpc::IndexParam& index_param); + CreateIndex(const grpc::IndexParam &index_param); void - Insert(grpc::VectorIds& vector_ids, - const grpc::InsertParam& insert_param, - Status& status); + Insert(grpc::VectorIds &vector_ids, + const grpc::InsertParam &insert_param, + Status &status); Status - Search(::milvus::grpc::TopKQueryResultList& topk_query_result_list, + Search(::milvus::grpc::TopKQueryResultList &topk_query_result_list, const grpc::SearchParam &search_param); Status - DescribeTable(grpc::TableSchema& grpc_schema, - const std::string& table_name); + DescribeTable(grpc::TableSchema &grpc_schema, + const std::string &table_name); int64_t - CountTable(const std::string& table_name, Status& status); + CountTable(const std::string &table_name, Status &status); Status ShowTables(milvus::grpc::TableNameList &table_name_list); Status - Cmd(std::string &result, const std::string& cmd); + Cmd(std::string &result, const std::string &cmd); Status DeleteByRange(grpc::DeleteByRangeParam &delete_by_range_param); @@ -93,8 +91,8 @@ public: Status Disconnect(); -private: + private: std::unique_ptr stub_; }; -} +} // namespace milvus diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index 05de495fbc..e6025fd52e 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -28,7 +28,6 @@ */ namespace milvus { - /** * @brief Index Type */ @@ -108,7 +107,6 @@ struct IndexParam { */ class Connection { public: - /** * @brief CreateConnection * @@ -131,7 +129,7 @@ class Connection { */ static Status - Destroy(std::shared_ptr& connection_ptr); + Destroy(std::shared_ptr &connection_ptr); /** * @brief Connect @@ -180,7 +178,6 @@ class Connection { virtual Status Disconnect() = 0; - /** * @brief Create table method * @@ -193,7 +190,6 @@ class Connection { virtual Status CreateTable(const TableSchema ¶m) = 0; - /** * @brief Test table existence method * @@ -206,7 +202,6 @@ class Connection { virtual bool HasTable(const std::string &table_name) = 0; - /** * @brief Delete table method * @@ -248,8 +243,8 @@ class Connection { */ virtual Status Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) = 0; + const std::vector &record_array, + std::vector &id_array) = 0; /** * @brief Search vector @@ -266,11 +261,11 @@ class Connection { */ virtual Status Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) = 0; + const std::vector &query_record_array, + const std::vector &query_range_array, + int64_t topk, + int64_t nprobe, + std::vector &topk_query_result_array) = 0; /** * @brief Show table description @@ -297,7 +292,7 @@ class Connection { */ virtual Status CountTable(const std::string &table_name, - int64_t &row_count) = 0; + int64_t &row_count) = 0; /** * @brief Show all tables in database @@ -395,4 +390,4 @@ class Connection { DropIndex(const std::string &table_name) const = 0; }; -} \ No newline at end of file +} // namespace milvus diff --git a/cpp/src/sdk/include/Status.h b/cpp/src/sdk/include/Status.h index cdb5f54ede..670d9662b4 100644 --- a/cpp/src/sdk/include/Status.h +++ b/cpp/src/sdk/include/Status.h @@ -29,6 +29,7 @@ namespace milvus { */ enum class StatusCode { OK = 0, + // system error section UnknownError = 1, NotSupported, @@ -44,7 +45,7 @@ enum class StatusCode { * @brief Status for SDK interface return */ class Status { -public: + public: Status(StatusCode code, const std::string &msg); Status(); ~Status(); @@ -60,28 +61,32 @@ public: operator=(Status &&s); static Status - OK() { return Status(); } + OK() { + return Status(); + } bool - ok() const { return state_ == nullptr || code() == StatusCode::OK; } + ok() const { + return state_ == nullptr || code() == StatusCode::OK; + } StatusCode code() const { - return (state_ == nullptr) ? StatusCode::OK : *(StatusCode*)(state_); + return (state_ == nullptr) ? StatusCode::OK : *(StatusCode *) (state_); } std::string message() const; -private: + private: inline void CopyFrom(const Status &s); inline void MoveFrom(Status &s); -private: + private: const char *state_ = nullptr; }; // Status -} //Milvus \ No newline at end of file +} // namespace milvus diff --git a/cpp/src/sdk/interface/ConnectionImpl.cpp b/cpp/src/sdk/interface/ConnectionImpl.cpp index 3730cfd191..5a01cd1a79 100644 --- a/cpp/src/sdk/interface/ConnectionImpl.cpp +++ b/cpp/src/sdk/interface/ConnectionImpl.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "ConnectionImpl.h" +#include "sdk/interface/ConnectionImpl.h" namespace milvus { @@ -25,7 +25,7 @@ Connection::Create() { } Status -Connection::Destroy(std::shared_ptr& connection_ptr) { +Connection::Destroy(std::shared_ptr &connection_ptr) { if (connection_ptr != nullptr) { return connection_ptr->Disconnect(); } @@ -84,19 +84,18 @@ ConnectionImpl::CreateIndex(const IndexParam &index_param) { Status ConnectionImpl::Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) { + const std::vector &record_array, + std::vector &id_array) { return client_proxy_->Insert(table_name, record_array, id_array); } - Status ConnectionImpl::Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) { + const std::vector &query_record_array, + const std::vector &query_range_array, + int64_t topk, + int64_t nprobe, + std::vector &topk_query_result_array) { return client_proxy_->Search(table_name, query_record_array, query_range_array, topk, nprobe, topk_query_result_array); } @@ -133,7 +132,7 @@ ConnectionImpl::DumpTaskTables() const { Status ConnectionImpl::DeleteByRange(Range &range, - const std::string &table_name) { + const std::string &table_name) { return client_proxy_->DeleteByRange(range, table_name); } @@ -143,7 +142,7 @@ ConnectionImpl::PreloadTable(const std::string &table_name) const { } Status -ConnectionImpl::DescribeIndex(const std::string &table_name, IndexParam& index_param) const { +ConnectionImpl::DescribeIndex(const std::string &table_name, IndexParam &index_param) const { return client_proxy_->DescribeIndex(table_name, index_param); } @@ -152,4 +151,4 @@ ConnectionImpl::DropIndex(const std::string &table_name) const { return client_proxy_->DropIndex(table_name); } -} +} // namespace milvus diff --git a/cpp/src/sdk/interface/ConnectionImpl.h b/cpp/src/sdk/interface/ConnectionImpl.h index 6a402fcb0d..7d5e090882 100644 --- a/cpp/src/sdk/interface/ConnectionImpl.h +++ b/cpp/src/sdk/interface/ConnectionImpl.h @@ -18,88 +18,92 @@ #pragma once #include "MilvusApi.h" -#include "src/sdk/grpc/ClientProxy.h" +#include "sdk/grpc/ClientProxy.h" + +#include +#include +#include namespace milvus { class ConnectionImpl : public Connection { -public: + public: ConnectionImpl(); // Implementations of the Connection interface - virtual Status + Status Connect(const ConnectParam ¶m) override; - virtual Status + Status Connect(const std::string &uri) override; - virtual Status + Status Connected() const override; - virtual Status + Status Disconnect() override; - virtual Status + Status CreateTable(const TableSchema ¶m) override; - virtual - bool HasTable(const std::string &table_name) override; + bool + HasTable(const std::string &table_name) override; - virtual Status + Status DropTable(const std::string &table_name) override; - virtual Status + Status CreateIndex(const IndexParam &index_param) override; - virtual Status + Status Insert(const std::string &table_name, const std::vector &record_array, std::vector &id_array) override; - virtual Status + Status Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) override; + const std::vector &query_record_array, + const std::vector &query_range_array, + int64_t topk, + int64_t nprobe, + std::vector &topk_query_result_array) override; - virtual Status + Status DescribeTable(const std::string &table_name, TableSchema &table_schema) override; - virtual Status + Status CountTable(const std::string &table_name, int64_t &row_count) override; - virtual Status + Status ShowTables(std::vector &table_array) override; - virtual std::string + std::string ClientVersion() const override; - virtual std::string + std::string ServerVersion() const override; - virtual std::string + std::string ServerStatus() const override; - virtual std::string + std::string DumpTaskTables() const override; - virtual Status + Status DeleteByRange(Range &range, const std::string &table_name) override; - virtual Status + Status PreloadTable(const std::string &table_name) const override; - virtual Status - DescribeIndex(const std::string &table_name, IndexParam& index_param) const override; + Status + DescribeIndex(const std::string &table_name, IndexParam &index_param) const override; - virtual Status + Status DropIndex(const std::string &table_name) const override; -private: + private: std::shared_ptr client_proxy_; }; -} +} // namespace milvus diff --git a/cpp/src/sdk/interface/Status.cpp b/cpp/src/sdk/interface/Status.cpp index e6fd06c72e..a8780f2ddd 100644 --- a/cpp/src/sdk/interface/Status.cpp +++ b/cpp/src/sdk/interface/Status.cpp @@ -23,12 +23,12 @@ namespace milvus { constexpr int CODE_WIDTH = sizeof(StatusCode); -Status::Status(StatusCode code, const std::string& msg) { +Status::Status(StatusCode code, const std::string &msg) { //4 bytes store code //4 bytes store message length //the left bytes store message string - const uint32_t length = (uint32_t)msg.size(); - char* result = new char[length + sizeof(length) + CODE_WIDTH]; + const uint32_t length = (uint32_t) msg.size(); + char *result = new char[length + sizeof(length) + CODE_WIDTH]; memcpy(result, &code, CODE_WIDTH); memcpy(result + CODE_WIDTH, &length, sizeof(length)); memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length); @@ -37,8 +37,7 @@ Status::Status(StatusCode code, const std::string& msg) { } Status::Status() - : state_(nullptr) { - + : state_(nullptr) { } Status::~Status() { @@ -46,22 +45,22 @@ Status::~Status() { } Status::Status(const Status &s) - : state_(nullptr) { + : state_(nullptr) { CopyFrom(s); } -Status& +Status & Status::operator=(const Status &s) { CopyFrom(s); return *this; } Status::Status(Status &&s) - : state_(nullptr) { + : state_(nullptr) { MoveFrom(s); } -Status& +Status & Status::operator=(Status &&s) { MoveFrom(s); return *this; @@ -71,7 +70,7 @@ void Status::CopyFrom(const Status &s) { delete state_; state_ = nullptr; - if(s.state_ == nullptr) { + if (s.state_ == nullptr) { return; } @@ -79,7 +78,7 @@ Status::CopyFrom(const Status &s) { memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); int buff_len = length + sizeof(length) + CODE_WIDTH; state_ = new char[buff_len]; - memcpy((void*)state_, (void*)s.state_, buff_len); + memcpy((void *) state_, (void *) s.state_, buff_len); } void @@ -98,12 +97,13 @@ Status::message() const { std::string msg; uint32_t length = 0; memcpy(&length, state_ + CODE_WIDTH, sizeof(length)); - if(length > 0) { + if (length > 0) { msg.append(state_ + sizeof(length) + CODE_WIDTH, length); } return msg; } -} +} // namespace milvus + diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 8ee2f7c62b..d0b2b84539 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "Config.h" +#include "server/Config.h" #include #include @@ -23,12 +23,13 @@ #include #include #include +#include +#include #include "config/ConfigMgr.h" #include "utils/CommonUtil.h" #include "utils/ValidationUtil.h" - namespace zilliz { namespace milvus { namespace server { @@ -70,12 +71,196 @@ Config::LoadConfigFile(const std::string &filename) { return Status::OK(); } +Status +Config::ValidateConfig() { + Status s; + + /* server config */ + std::string server_addr; + s = GetServerConfigAddress(server_addr); + if (!s.ok()) return s; + + std::string server_port; + s = GetServerConfigPort(server_port); + if (!s.ok()) return s; + + std::string server_mode; + s = GetServerConfigDeployMode(server_mode); + if (!s.ok()) return s; + + std::string server_time_zone; + s = GetServerConfigTimeZone(server_time_zone); + if (!s.ok()) return s; + + /* db config */ + std::string db_primary_path; + s = GetDBConfigPrimaryPath(db_primary_path); + if (!s.ok()) return s; + + std::string db_secondary_path; + s = GetDBConfigSecondaryPath(db_secondary_path); + if (!s.ok()) return s; + + std::string db_backend_url; + s = GetDBConfigBackendUrl(db_backend_url); + if (!s.ok()) return s; + + int32_t db_archive_disk_threshold; + s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold); + if (!s.ok()) return s; + + int32_t db_archive_days_threshold; + s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold); + if (!s.ok()) return s; + + int32_t db_insert_buffer_size; + s = GetDBConfigInsertBufferSize(db_insert_buffer_size); + if (!s.ok()) return s; + + int32_t db_build_index_gpu; + s = GetDBConfigBuildIndexGPU(db_build_index_gpu); + if (!s.ok()) return s; + + /* metric config */ + bool metric_enable_monitor; + s = GetMetricConfigEnableMonitor(metric_enable_monitor); + if (!s.ok()) return s; + + std::string metric_collector; + s = GetMetricConfigCollector(metric_collector); + if (!s.ok()) return s; + + std::string metric_prometheus_port; + s = GetMetricConfigPrometheusPort(metric_prometheus_port); + if (!s.ok()) return s; + + /* cache config */ + int32_t cache_cpu_mem_capacity; + s = GetCacheConfigCpuMemCapacity(cache_cpu_mem_capacity); + if (!s.ok()) return s; + + float cache_cpu_mem_threshold; + s = GetCacheConfigCpuMemThreshold(cache_cpu_mem_threshold); + if (!s.ok()) return s; + + int32_t cache_gpu_mem_capacity; + s = GetCacheConfigGpuMemCapacity(cache_gpu_mem_capacity); + if (!s.ok()) return s; + + float cache_gpu_mem_threshold; + s = GetCacheConfigGpuMemThreshold(cache_gpu_mem_threshold); + if (!s.ok()) return s; + + bool cache_insert_data; + s = GetCacheConfigCacheInsertData(cache_insert_data); + if (!s.ok()) return s; + + /* engine config */ + int32_t engine_blas_threshold; + s = GetEngineConfigBlasThreshold(engine_blas_threshold); + if (!s.ok()) return s; + + int32_t engine_omp_thread_num; + s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); + if (!s.ok()) return s; + + /* resource config */ + std::string resource_mode; + s = GetResourceConfigMode(resource_mode); + if (!s.ok()) return s; + + std::vector resource_pool; + s = GetResourceConfigPool(resource_pool); + if (!s.ok()) return s; + + return Status::OK(); +} + +Status +Config::ResetDefaultConfig() { + Status s; + + /* server config */ + s = SetServerConfigAddress(CONFIG_SERVER_ADDRESS_DEFAULT); + if (!s.ok()) return s; + + s = SetServerConfigPort(CONFIG_SERVER_PORT_DEFAULT); + if (!s.ok()) return s; + + s = SetServerConfigDeployMode(CONFIG_SERVER_DEPLOY_MODE_DEFAULT); + if (!s.ok()) return s; + + s = SetServerConfigTimeZone(CONFIG_SERVER_TIME_ZONE_DEFAULT); + if (!s.ok()) return s; + + /* db config */ + s = SetDBConfigPrimaryPath(CONFIG_DB_PRIMARY_PATH_DEFAULT); + if (!s.ok()) return s; + + s = SetDBConfigSecondaryPath(CONFIG_DB_SECONDARY_PATH_DEFAULT); + if (!s.ok()) return s; + + s = SetDBConfigBackendUrl(CONFIG_DB_BACKEND_URL_DEFAULT); + if (!s.ok()) return s; + + s = SetDBConfigArchiveDiskThreshold(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); + if (!s.ok()) return s; + + s = SetDBConfigArchiveDaysThreshold(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); + if (!s.ok()) return s; + + s = SetDBConfigInsertBufferSize(CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); + if (!s.ok()) return s; + + s = SetDBConfigBuildIndexGPU(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); + if (!s.ok()) return s; + + /* metric config */ + s = SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); + if (!s.ok()) return s; + + s = SetMetricConfigCollector(CONFIG_METRIC_COLLECTOR_DEFAULT); + if (!s.ok()) return s; + + s = SetMetricConfigPrometheusPort(CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); + if (!s.ok()) return s; + + /* cache config */ + s = SetCacheConfigCpuMemCapacity(CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); + if (!s.ok()) return s; + + s = SetCacheConfigCpuMemThreshold(CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); + if (!s.ok()) return s; + + s = SetCacheConfigGpuMemCapacity(CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); + if (!s.ok()) return s; + + s = SetCacheConfigGpuMemThreshold(CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); + if (!s.ok()) return s; + + s = SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); + if (!s.ok()) return s; + + /* engine config */ + s = SetEngineConfigBlasThreshold(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); + if (!s.ok()) return s; + + s = SetEngineConfigOmpThreadNum(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); + if (!s.ok()) return s; + + /* resource config */ + s = SetResourceConfigMode(CONFIG_RESOURCE_MODE_DEFAULT); + if (!s.ok()) return s; + + return Status::OK(); +} + void -Config::PrintConfigSection(const std::string& config_node_name) { +Config::PrintConfigSection(const std::string &config_node_name) { std::cout << std::endl; std::cout << config_node_name << ":" << std::endl; if (config_map_.find(config_node_name) != config_map_.end()) { - for (auto item: config_map_[config_node_name]) { + for (auto item : config_map_[config_node_name]) { std::cout << item.first << ": " << item.second << std::endl; } } @@ -114,9 +299,12 @@ Config::CheckServerConfigPort(const std::string &value) { } Status -Config::CheckServerConfigMode(const std::string &value) { - if (value != "single" && value != "cluster" && value != "read_only") { - return Status(SERVER_INVALID_ARGUMENT, "Invalid server config mode [single, cluster, read_only]: " + value); +Config::CheckServerConfigDeployMode(const std::string &value) { + if (value != "single" && + value != "cluster_readonly" && + value != "cluster_writable") { + return Status(SERVER_INVALID_ARGUMENT, + "Invalid server config mode [single, cluster_readonly, cluster_writable]: " + value); } return Status::OK(); } @@ -140,15 +328,15 @@ Config::CheckServerConfigTimeZone(const std::string &value) { } Status -Config::CheckDBConfigPath(const std::string &value) { +Config::CheckDBConfigPrimaryPath(const std::string &value) { if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, "DB config path empty"); + return Status(SERVER_INVALID_ARGUMENT, "DB config primary_path empty"); } return Status::OK(); } Status -Config::CheckDBConfigSlavePath(const std::string &value) { +Config::CheckDBConfigSecondaryPath(const std::string &value) { return Status::OK(); } @@ -177,15 +365,15 @@ Config::CheckDBConfigArchiveDaysThreshold(const std::string &value) { } Status -Config::CheckDBConfigBufferSize(const std::string &value) { +Config::CheckDBConfigInsertBufferSize(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config buffer_size: " + value); + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config insert_buffer_size: " + value); } else { int64_t buffer_size = std::stoi(value) * GB; - unsigned long total_mem = 0, free_mem = 0; + uint64_t total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (buffer_size >= total_mem) { - return Status(SERVER_INVALID_ARGUMENT, "DB config buffer_size exceed system memory: " + value); + return Status(SERVER_INVALID_ARGUMENT, "DB config insert_buffer_size exceed system memory: " + value); } } return Status::OK(); @@ -205,7 +393,7 @@ Config::CheckDBConfigBuildIndexGPU(const std::string &value) { } Status -Config::CheckMetricConfigAutoBootup(const std::string& value) { +Config::CheckMetricConfigEnableMonitor(const std::string &value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config auto_bootup: " + value); } @@ -213,7 +401,7 @@ Config::CheckMetricConfigAutoBootup(const std::string& value) { } Status -Config::CheckMetricConfigCollector(const std::string& value) { +Config::CheckMetricConfigCollector(const std::string &value) { if (value != "prometheus") { return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config collector: " + value); } @@ -221,7 +409,7 @@ Config::CheckMetricConfigCollector(const std::string& value) { } Status -Config::CheckMetricConfigPrometheusPort(const std::string& value) { +Config::CheckMetricConfigPrometheusPort(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config prometheus_port: " + value); } @@ -229,12 +417,12 @@ Config::CheckMetricConfigPrometheusPort(const std::string& value) { } Status -Config::CheckCacheConfigCpuMemCapacity(const std::string& value) { +Config::CheckCacheConfigCpuMemCapacity(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_capacity: " + value); } else { uint64_t cpu_cache_capacity = std::stoi(value) * GB; - unsigned long total_mem = 0, free_mem = 0; + uint64_t total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (cpu_cache_capacity >= total_mem) { return Status(SERVER_INVALID_ARGUMENT, "Cache config cpu_mem_capacity exceed system memory: " + value); @@ -242,10 +430,10 @@ Config::CheckCacheConfigCpuMemCapacity(const std::string& value) { std::cerr << "Warning: cpu_mem_capacity value is too big" << std::endl; } - int32_t buffer_size; - Status s = GetDBConfigBufferSize(buffer_size); + int32_t buffer_value; + Status s = GetDBConfigInsertBufferSize(buffer_value); if (!s.ok()) return s; - int64_t insert_buffer_size = buffer_size * GB; + int64_t insert_buffer_size = buffer_value * GB; if (insert_buffer_size + cpu_cache_capacity >= total_mem) { return Status(SERVER_INVALID_ARGUMENT, "Sum of cpu_mem_capacity and buffer_size exceed system memory"); } @@ -254,7 +442,7 @@ Config::CheckCacheConfigCpuMemCapacity(const std::string& value) { } Status -Config::CheckCacheConfigCpuMemThreshold(const std::string& value) { +Config::CheckCacheConfigCpuMemThreshold(const std::string &value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_threshold: " + value); } else { @@ -267,7 +455,7 @@ Config::CheckCacheConfigCpuMemThreshold(const std::string& value) { } Status -Config::CheckCacheConfigGpuMemCapacity(const std::string& value) { +Config::CheckCacheConfigGpuMemCapacity(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { std::cerr << "ERROR: gpu_cache_capacity " << value << " is not a number" << std::endl; } else { @@ -290,7 +478,7 @@ Config::CheckCacheConfigGpuMemCapacity(const std::string& value) { } Status -Config::CheckCacheConfigGpuMemThreshold(const std::string& value) { +Config::CheckCacheConfigGpuMemThreshold(const std::string &value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_mem_threshold: " + value); } else { @@ -303,7 +491,7 @@ Config::CheckCacheConfigGpuMemThreshold(const std::string& value) { } Status -Config::CheckCacheConfigCacheInsertData(const std::string& value) { +Config::CheckCacheConfigCacheInsertData(const std::string &value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cache_insert_data: " + value); } @@ -311,7 +499,7 @@ Config::CheckCacheConfigCacheInsertData(const std::string& value) { } Status -Config::CheckEngineConfigBlasThreshold(const std::string& value) { +Config::CheckEngineConfigBlasThreshold(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config blas threshold: " + value); } @@ -319,7 +507,7 @@ Config::CheckEngineConfigBlasThreshold(const std::string& value) { } Status -Config::CheckEngineConfigOmpThreadNum(const std::string& value) { +Config::CheckEngineConfigOmpThreadNum(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); } else { @@ -333,7 +521,7 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { } Status -Config::CheckResourceConfigMode(const std::string& value) { +Config::CheckResourceConfigMode(const std::string &value) { if (value != "simple") { return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config mode: " + value); } @@ -341,7 +529,7 @@ Config::CheckResourceConfigMode(const std::string& value) { } Status -Config::CheckResourceConfigPool(const std::vector& value) { +Config::CheckResourceConfigPool(const std::vector &value) { if (value.empty()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config pool"); } @@ -403,12 +591,12 @@ Config::GetServerConfigStrPort() { } std::string -Config::GetServerConfigStrMode() { +Config::GetServerConfigStrDeployMode() { std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_MODE, - CONFIG_SERVER_MODE_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value).ok()) { + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_DEPLOY_MODE, + CONFIG_SERVER_DEPLOY_MODE_DEFAULT); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); } return value; } @@ -427,23 +615,23 @@ Config::GetServerConfigStrTimeZone() { //////////////////////////////////////////////////////////////////////////////// /* db config */ std::string -Config::GetDBConfigStrPath() { +Config::GetDBConfigStrPrimaryPath() { std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PATH, - CONFIG_DB_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value).ok()) { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PRIMARY_PATH, + CONFIG_DB_PRIMARY_PATH_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); } return value; } std::string -Config::GetDBConfigStrSlavePath() { +Config::GetDBConfigStrSecondaryPath() { std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SLAVE_PATH, - CONFIG_DB_SLAVE_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value).ok()) { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SECONDARY_PATH, + CONFIG_DB_SECONDARY_PATH_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); } return value; } @@ -482,12 +670,12 @@ Config::GetDBConfigStrArchiveDaysThreshold() { } std::string -Config::GetDBConfigStrBufferSize() { +Config::GetDBConfigStrInsertBufferSize() { std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUFFER_SIZE, - CONFIG_DB_BUFFER_SIZE_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value).ok()) { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, + CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); } return value; } @@ -506,12 +694,12 @@ Config::GetDBConfigStrBuildIndexGPU() { //////////////////////////////////////////////////////////////////////////////// /* metric config */ std::string -Config::GetMetricConfigStrAutoBootup() { +Config::GetMetricConfigStrEnableMonitor() { std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { - value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_AUTO_BOOTUP, - CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value).ok()) { + value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_ENABLE_MONITOR, + CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); } return value; } @@ -632,52 +820,51 @@ Config::GetResourceConfigStrMode() { return value; } - //////////////////////////////////////////////////////////////////////////////// Status -Config::GetServerConfigAddress(std::string& value) { +Config::GetServerConfigAddress(std::string &value) { value = GetServerConfigStrAddress(); return CheckServerConfigAddress(value); } Status -Config::GetServerConfigPort(std::string& value) { +Config::GetServerConfigPort(std::string &value) { value = GetServerConfigStrPort(); return CheckServerConfigPort(value); } Status -Config::GetServerConfigMode(std::string& value) { - value = GetServerConfigStrMode(); - return CheckServerConfigMode(value); +Config::GetServerConfigDeployMode(std::string &value) { + value = GetServerConfigStrDeployMode(); + return CheckServerConfigDeployMode(value); } Status -Config::GetServerConfigTimeZone(std::string& value) { +Config::GetServerConfigTimeZone(std::string &value) { value = GetServerConfigStrTimeZone(); return CheckServerConfigTimeZone(value); } Status -Config::GetDBConfigPath(std::string& value) { - value = GetDBConfigStrPath(); - return CheckDBConfigPath(value); +Config::GetDBConfigPrimaryPath(std::string &value) { + value = GetDBConfigStrPrimaryPath(); + return CheckDBConfigPrimaryPath(value); } Status -Config::GetDBConfigSlavePath(std::string& value) { - value = GetDBConfigStrSlavePath(); +Config::GetDBConfigSecondaryPath(std::string &value) { + value = GetDBConfigStrSecondaryPath(); return Status::OK(); } Status -Config::GetDBConfigBackendUrl(std::string& value) { +Config::GetDBConfigBackendUrl(std::string &value) { value = GetDBConfigStrBackendUrl(); return CheckDBConfigBackendUrl(value); } Status -Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { +Config::GetDBConfigArchiveDiskThreshold(int32_t &value) { std::string str = GetDBConfigStrArchiveDiskThreshold(); Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) return s; @@ -686,7 +873,7 @@ Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { } Status -Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { +Config::GetDBConfigArchiveDaysThreshold(int32_t &value) { std::string str = GetDBConfigStrArchiveDaysThreshold(); Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) return s; @@ -695,16 +882,16 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { } Status -Config::GetDBConfigBufferSize(int32_t& value) { - std::string str = GetDBConfigStrBufferSize(); - Status s = CheckDBConfigBufferSize(str); +Config::GetDBConfigInsertBufferSize(int32_t &value) { + std::string str = GetDBConfigStrInsertBufferSize(); + Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); } Status -Config::GetDBConfigBuildIndexGPU(int32_t& value) { +Config::GetDBConfigBuildIndexGPU(int32_t &value) { std::string str = GetDBConfigStrBuildIndexGPU(); Status s = CheckDBConfigBuildIndexGPU(str); if (!s.ok()) return s; @@ -713,9 +900,9 @@ Config::GetDBConfigBuildIndexGPU(int32_t& value) { } Status -Config::GetMetricConfigAutoBootup(bool& value) { - std::string str = GetMetricConfigStrAutoBootup(); - Status s = CheckMetricConfigPrometheusPort(str); +Config::GetMetricConfigEnableMonitor(bool &value) { + std::string str = GetMetricConfigStrEnableMonitor(); + Status s = CheckMetricConfigEnableMonitor(str); if (!s.ok()) return s; std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); @@ -723,19 +910,19 @@ Config::GetMetricConfigAutoBootup(bool& value) { } Status -Config::GetMetricConfigCollector(std::string& value) { +Config::GetMetricConfigCollector(std::string &value) { value = GetMetricConfigStrCollector(); return Status::OK(); } Status -Config::GetMetricConfigPrometheusPort(std::string& value) { +Config::GetMetricConfigPrometheusPort(std::string &value) { value = GetMetricConfigStrPrometheusPort(); return CheckMetricConfigPrometheusPort(value); } Status -Config::GetCacheConfigCpuMemCapacity(int32_t& value) { +Config::GetCacheConfigCpuMemCapacity(int32_t &value) { std::string str = GetCacheConfigStrCpuMemCapacity(); Status s = CheckCacheConfigCpuMemCapacity(str); if (!s.ok()) return s; @@ -744,7 +931,7 @@ Config::GetCacheConfigCpuMemCapacity(int32_t& value) { } Status -Config::GetCacheConfigCpuMemThreshold(float& value) { +Config::GetCacheConfigCpuMemThreshold(float &value) { std::string str = GetCacheConfigStrCpuMemThreshold(); Status s = CheckCacheConfigCpuMemThreshold(str); if (!s.ok()) return s; @@ -753,7 +940,7 @@ Config::GetCacheConfigCpuMemThreshold(float& value) { } Status -Config::GetCacheConfigGpuMemCapacity(int32_t& value) { +Config::GetCacheConfigGpuMemCapacity(int32_t &value) { std::string str = GetCacheConfigStrGpuMemCapacity(); Status s = CheckCacheConfigGpuMemCapacity(str); if (!s.ok()) return s; @@ -762,7 +949,7 @@ Config::GetCacheConfigGpuMemCapacity(int32_t& value) { } Status -Config::GetCacheConfigGpuMemThreshold(float& value) { +Config::GetCacheConfigGpuMemThreshold(float &value) { std::string str = GetCacheConfigStrGpuMemThreshold(); Status s = CheckCacheConfigGpuMemThreshold(str); if (!s.ok()) return s; @@ -771,7 +958,7 @@ Config::GetCacheConfigGpuMemThreshold(float& value) { } Status -Config::GetCacheConfigCacheInsertData(bool& value) { +Config::GetCacheConfigCacheInsertData(bool &value) { std::string str = GetCacheConfigStrCacheInsertData(); Status s = CheckCacheConfigCacheInsertData(str); if (!s.ok()) return s; @@ -781,7 +968,7 @@ Config::GetCacheConfigCacheInsertData(bool& value) { } Status -Config::GetEngineConfigBlasThreshold(int32_t& value) { +Config::GetEngineConfigBlasThreshold(int32_t &value) { std::string str = GetEngineConfigStrBlasThreshold(); Status s = CheckEngineConfigBlasThreshold(str); if (!s.ok()) return s; @@ -790,7 +977,7 @@ Config::GetEngineConfigBlasThreshold(int32_t& value) { } Status -Config::GetEngineConfigOmpThreadNum(int32_t& value) { +Config::GetEngineConfigOmpThreadNum(int32_t &value) { std::string str = GetEngineConfigStrOmpThreadNum(); Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) return s; @@ -799,13 +986,13 @@ Config::GetEngineConfigOmpThreadNum(int32_t& value) { } Status -Config::GetResourceConfigMode(std::string& value) { +Config::GetResourceConfigMode(std::string &value) { value = GetResourceConfigStrMode(); return CheckResourceConfigMode(value); } Status -Config::GetResourceConfigPool(std::vector& value) { +Config::GetResourceConfigPool(std::vector &value) { ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE); value = resource_config.GetSequence(CONFIG_RESOURCE_POOL); return CheckResourceConfigPool(value); @@ -814,7 +1001,7 @@ Config::GetResourceConfigPool(std::vector& value) { /////////////////////////////////////////////////////////////////////////////// /* server config */ Status -Config::SetServerConfigAddress(const std::string& value) { +Config::SetServerConfigAddress(const std::string &value) { Status s = CheckServerConfigAddress(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); @@ -822,7 +1009,7 @@ Config::SetServerConfigAddress(const std::string& value) { } Status -Config::SetServerConfigPort(const std::string& value) { +Config::SetServerConfigPort(const std::string &value) { Status s = CheckServerConfigPort(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); @@ -830,15 +1017,15 @@ Config::SetServerConfigPort(const std::string& value) { } Status -Config::SetServerConfigMode(const std::string& value) { - Status s = CheckServerConfigMode(value); +Config::SetServerConfigDeployMode(const std::string &value) { + Status s = CheckServerConfigDeployMode(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); return Status::OK(); } Status -Config::SetServerConfigTimeZone(const std::string& value) { +Config::SetServerConfigTimeZone(const std::string &value) { Status s = CheckServerConfigTimeZone(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); @@ -847,23 +1034,23 @@ Config::SetServerConfigTimeZone(const std::string& value) { /* db config */ Status -Config::SetDBConfigPath(const std::string& value) { - Status s = CheckDBConfigPath(value); +Config::SetDBConfigPrimaryPath(const std::string &value) { + Status s = CheckDBConfigPrimaryPath(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); return Status::OK(); } Status -Config::SetDBConfigSlavePath(const std::string& value) { - Status s = CheckDBConfigSlavePath(value); +Config::SetDBConfigSecondaryPath(const std::string &value) { + Status s = CheckDBConfigSecondaryPath(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); return Status::OK(); } Status -Config::SetDBConfigBackendUrl(const std::string& value) { +Config::SetDBConfigBackendUrl(const std::string &value) { Status s = CheckDBConfigBackendUrl(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); @@ -871,7 +1058,7 @@ Config::SetDBConfigBackendUrl(const std::string& value) { } Status -Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { +Config::SetDBConfigArchiveDiskThreshold(const std::string &value) { Status s = CheckDBConfigArchiveDiskThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); @@ -879,7 +1066,7 @@ Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { } Status -Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { +Config::SetDBConfigArchiveDaysThreshold(const std::string &value) { Status s = CheckDBConfigArchiveDaysThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); @@ -887,15 +1074,15 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { } Status -Config::SetDBConfigBufferSize(const std::string& value) { - Status s = CheckDBConfigBufferSize(value); +Config::SetDBConfigInsertBufferSize(const std::string &value) { + Status s = CheckDBConfigInsertBufferSize(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); return Status::OK(); } Status -Config::SetDBConfigBuildIndexGPU(const std::string& value) { +Config::SetDBConfigBuildIndexGPU(const std::string &value) { Status s = CheckDBConfigBuildIndexGPU(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); @@ -904,15 +1091,15 @@ Config::SetDBConfigBuildIndexGPU(const std::string& value) { /* metric config */ Status -Config::SetMetricConfigAutoBootup(const std::string& value) { - Status s = CheckMetricConfigAutoBootup(value); +Config::SetMetricConfigEnableMonitor(const std::string &value) { + Status s = CheckMetricConfigEnableMonitor(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_AUTO_BOOTUP, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value); return Status::OK(); } Status -Config::SetMetricConfigCollector(const std::string& value) { +Config::SetMetricConfigCollector(const std::string &value) { Status s = CheckMetricConfigCollector(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_COLLECTOR, value); @@ -920,7 +1107,7 @@ Config::SetMetricConfigCollector(const std::string& value) { } Status -Config::SetMetricConfigPrometheusPort(const std::string& value) { +Config::SetMetricConfigPrometheusPort(const std::string &value) { Status s = CheckMetricConfigPrometheusPort(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_PROMETHEUS_PORT, value); @@ -929,7 +1116,7 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) { /* cache config */ Status -Config::SetCacheConfigCpuMemCapacity(const std::string& value) { +Config::SetCacheConfigCpuMemCapacity(const std::string &value) { Status s = CheckCacheConfigCpuMemCapacity(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_MEM_CAPACITY, value); @@ -937,7 +1124,7 @@ Config::SetCacheConfigCpuMemCapacity(const std::string& value) { } Status -Config::SetCacheConfigCpuMemThreshold(const std::string& value) { +Config::SetCacheConfigCpuMemThreshold(const std::string &value) { Status s = CheckCacheConfigCpuMemThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); @@ -945,7 +1132,7 @@ Config::SetCacheConfigCpuMemThreshold(const std::string& value) { } Status -Config::SetCacheConfigGpuMemCapacity(const std::string& value) { +Config::SetCacheConfigGpuMemCapacity(const std::string &value) { Status s = CheckCacheConfigGpuMemCapacity(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_MEM_CAPACITY, value); @@ -953,7 +1140,7 @@ Config::SetCacheConfigGpuMemCapacity(const std::string& value) { } Status -Config::SetCacheConfigGpuMemThreshold(const std::string& value) { +Config::SetCacheConfigGpuMemThreshold(const std::string &value) { Status s = CheckCacheConfigGpuMemThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); @@ -961,7 +1148,7 @@ Config::SetCacheConfigGpuMemThreshold(const std::string& value) { } Status -Config::SetCacheConfigCacheInsertData(const std::string& value) { +Config::SetCacheConfigCacheInsertData(const std::string &value) { Status s = CheckCacheConfigCacheInsertData(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CACHE_INSERT_DATA, value); @@ -970,7 +1157,7 @@ Config::SetCacheConfigCacheInsertData(const std::string& value) { /* engine config */ Status -Config::SetEngineConfigBlasThreshold(const std::string& value) { +Config::SetEngineConfigBlasThreshold(const std::string &value) { Status s = CheckEngineConfigBlasThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_BLAS_THRESHOLD, value); @@ -978,7 +1165,7 @@ Config::SetEngineConfigBlasThreshold(const std::string& value) { } Status -Config::SetEngineConfigOmpThreadNum(const std::string& value) { +Config::SetEngineConfigOmpThreadNum(const std::string &value) { Status s = CheckEngineConfigOmpThreadNum(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_OMP_THREAD_NUM, value); @@ -987,13 +1174,14 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) { /* resource config */ Status -Config::SetResourceConfigMode(const std::string& value) { +Config::SetResourceConfigMode(const std::string &value) { Status s = CheckResourceConfigMode(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_MODE, value); return Status::OK(); } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz + diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 956c08100f..af23dd67b6 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -17,155 +17,158 @@ #pragma once +#include +#include #include #include -#include "yaml-cpp/yaml.h" +#include + #include "utils/Status.h" #include "config/ConfigNode.h" - namespace zilliz { namespace milvus { namespace server { /* server config */ -static const char* CONFIG_SERVER = "server_config"; -static const char* CONFIG_SERVER_ADDRESS = "address"; -static const char* CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; -static const char* CONFIG_SERVER_PORT = "port"; -static const char* CONFIG_SERVER_PORT_DEFAULT = "19530"; -static const char* CONFIG_SERVER_MODE = "mode"; -static const char* CONFIG_SERVER_MODE_DEFAULT = "single"; -static const char* CONFIG_SERVER_TIME_ZONE = "time_zone"; -static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; +static const char *CONFIG_SERVER = "server_config"; +static const char *CONFIG_SERVER_ADDRESS = "address"; +static const char *CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; +static const char *CONFIG_SERVER_PORT = "port"; +static const char *CONFIG_SERVER_PORT_DEFAULT = "19530"; +static const char *CONFIG_SERVER_DEPLOY_MODE = "deploy_mode"; +static const char *CONFIG_SERVER_DEPLOY_MODE_DEFAULT = "single"; +static const char *CONFIG_SERVER_TIME_ZONE = "time_zone"; +static const char *CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; /* db config */ -static const char* CONFIG_DB = "db_config"; -static const char* CONFIG_DB_PATH = "path"; -static const char* CONFIG_DB_PATH_DEFAULT = "/tmp/milvus"; -static const char* CONFIG_DB_SLAVE_PATH = "slave_path"; -static const char* CONFIG_DB_SLAVE_PATH_DEFAULT = ""; -static const char* CONFIG_DB_BACKEND_URL = "backend_url"; -static const char* CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; -static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; -static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; -static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; -static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; -static const char* CONFIG_DB_BUFFER_SIZE = "buffer_size"; -static const char* CONFIG_DB_BUFFER_SIZE_DEFAULT = "4"; -static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; -static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; +static const char *CONFIG_DB = "db_config"; +static const char *CONFIG_DB_PRIMARY_PATH = "primary_path"; +static const char *CONFIG_DB_PRIMARY_PATH_DEFAULT = "/tmp/milvus"; +static const char *CONFIG_DB_SECONDARY_PATH = "secondary_path"; +static const char *CONFIG_DB_SECONDARY_PATH_DEFAULT = ""; +static const char *CONFIG_DB_BACKEND_URL = "backend_url"; +static const char *CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; +static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; +static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; +static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; +static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; +static const char *CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; +static const char *CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4"; +static const char *CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; +static const char *CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; /* cache config */ -static const char* CONFIG_CACHE = "cache_config"; -static const char* CONFIG_CACHE_CPU_MEM_CAPACITY = "cpu_mem_capacity"; -static const char* CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT = "16"; -static const char* CONFIG_CACHE_GPU_MEM_CAPACITY = "gpu_mem_capacity"; -static const char* CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT = "0"; -static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD = "cpu_mem_threshold"; -static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85"; -static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold"; -static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85"; -static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; -static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; +static const char *CONFIG_CACHE = "cache_config"; +static const char *CONFIG_CACHE_CPU_MEM_CAPACITY = "cpu_mem_capacity"; +static const char *CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT = "16"; +static const char *CONFIG_CACHE_GPU_MEM_CAPACITY = "gpu_mem_capacity"; +static const char *CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT = "0"; +static const char *CONFIG_CACHE_CPU_MEM_THRESHOLD = "cpu_mem_threshold"; +static const char *CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85"; +static const char *CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold"; +static const char *CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85"; +static const char *CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; +static const char *CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; /* metric config */ -static const char* CONFIG_METRIC = "metric_config"; -static const char* CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup"; -static const char* CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "false"; -static const char* CONFIG_METRIC_COLLECTOR = "collector"; -static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; -static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config"; -static const char* CONFIG_METRIC_PROMETHEUS_PORT = "port"; -static const char* CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "8080"; +static const char *CONFIG_METRIC = "metric_config"; +static const char *CONFIG_METRIC_ENABLE_MONITOR = "enable_monitor"; +static const char *CONFIG_METRIC_ENABLE_MONITOR_DEFAULT = "false"; +static const char *CONFIG_METRIC_COLLECTOR = "collector"; +static const char *CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; +static const char *CONFIG_METRIC_PROMETHEUS = "prometheus_config"; +static const char *CONFIG_METRIC_PROMETHEUS_PORT = "port"; +static const char *CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "8080"; /* engine config */ -static const char* CONFIG_ENGINE = "engine_config"; -static const char* CONFIG_ENGINE_BLAS_THRESHOLD = "blas_threshold"; -static const char* CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT = "20"; -static const char* CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num"; -static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; +static const char *CONFIG_ENGINE = "engine_config"; +static const char *CONFIG_ENGINE_BLAS_THRESHOLD = "blas_threshold"; +static const char *CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT = "20"; +static const char *CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num"; +static const char *CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; /* resource config */ -static const char* CONFIG_RESOURCE = "resource_config"; -static const char* CONFIG_RESOURCE_MODE = "mode"; -static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; -static const char* CONFIG_RESOURCE_POOL = "pool"; - +static const char *CONFIG_RESOURCE = "resource_config"; +static const char *CONFIG_RESOURCE_MODE = "mode"; +static const char *CONFIG_RESOURCE_MODE_DEFAULT = "simple"; +static const char *CONFIG_RESOURCE_POOL = "resource_pool"; class Config { public: - static Config& GetInstance(); - Status LoadConfigFile(const std::string& filename); + static Config &GetInstance(); + Status LoadConfigFile(const std::string &filename); + Status ValidateConfig(); + Status ResetDefaultConfig(); void PrintAll(); private: - ConfigNode& GetConfigNode(const std::string& name); + ConfigNode &GetConfigNode(const std::string &name); - Status GetConfigValueInMem(const std::string& parent_key, - const std::string& child_key, - std::string& value); + Status GetConfigValueInMem(const std::string &parent_key, + const std::string &child_key, + std::string &value); - void SetConfigValueInMem(const std::string& parent_key, - const std::string& child_key, - const std::string& value); + void SetConfigValueInMem(const std::string &parent_key, + const std::string &child_key, + const std::string &value); - void PrintConfigSection(const std::string& config_node_name); + void PrintConfigSection(const std::string &config_node_name); /////////////////////////////////////////////////////////////////////////// /* server config */ - Status CheckServerConfigAddress(const std::string& value); - Status CheckServerConfigPort(const std::string& value); - Status CheckServerConfigMode(const std::string& value); - Status CheckServerConfigTimeZone(const std::string& value); + Status CheckServerConfigAddress(const std::string &value); + Status CheckServerConfigPort(const std::string &value); + Status CheckServerConfigDeployMode(const std::string &value); + Status CheckServerConfigTimeZone(const std::string &value); /* db config */ - Status CheckDBConfigPath(const std::string& value); - Status CheckDBConfigSlavePath(const std::string& value); - Status CheckDBConfigBackendUrl(const std::string& value); - Status CheckDBConfigArchiveDiskThreshold(const std::string& value); - Status CheckDBConfigArchiveDaysThreshold(const std::string& value); - Status CheckDBConfigBufferSize(const std::string& value); - Status CheckDBConfigBuildIndexGPU(const std::string& value); + Status CheckDBConfigPrimaryPath(const std::string &value); + Status CheckDBConfigSecondaryPath(const std::string &value); + Status CheckDBConfigBackendUrl(const std::string &value); + Status CheckDBConfigArchiveDiskThreshold(const std::string &value); + Status CheckDBConfigArchiveDaysThreshold(const std::string &value); + Status CheckDBConfigInsertBufferSize(const std::string &value); + Status CheckDBConfigBuildIndexGPU(const std::string &value); /* metric config */ - Status CheckMetricConfigAutoBootup(const std::string& value); - Status CheckMetricConfigCollector(const std::string& value); - Status CheckMetricConfigPrometheusPort(const std::string& value); + Status CheckMetricConfigEnableMonitor(const std::string &value); + Status CheckMetricConfigCollector(const std::string &value); + Status CheckMetricConfigPrometheusPort(const std::string &value); /* cache config */ - Status CheckCacheConfigCpuMemCapacity(const std::string& value); - Status CheckCacheConfigCpuMemThreshold(const std::string& value); - Status CheckCacheConfigGpuMemCapacity(const std::string& value); - Status CheckCacheConfigGpuMemThreshold(const std::string& value); - Status CheckCacheConfigCacheInsertData(const std::string& value); + Status CheckCacheConfigCpuMemCapacity(const std::string &value); + Status CheckCacheConfigCpuMemThreshold(const std::string &value); + Status CheckCacheConfigGpuMemCapacity(const std::string &value); + Status CheckCacheConfigGpuMemThreshold(const std::string &value); + Status CheckCacheConfigCacheInsertData(const std::string &value); /* engine config */ - Status CheckEngineConfigBlasThreshold(const std::string& value); - Status CheckEngineConfigOmpThreadNum(const std::string& value); + Status CheckEngineConfigBlasThreshold(const std::string &value); + Status CheckEngineConfigOmpThreadNum(const std::string &value); /* resource config */ - Status CheckResourceConfigMode(const std::string& value); - Status CheckResourceConfigPool(const std::vector& value); + Status CheckResourceConfigMode(const std::string &value); + Status CheckResourceConfigPool(const std::vector &value); /////////////////////////////////////////////////////////////////////////// /* server config */ std::string GetServerConfigStrAddress(); std::string GetServerConfigStrPort(); - std::string GetServerConfigStrMode(); + std::string GetServerConfigStrDeployMode(); std::string GetServerConfigStrTimeZone(); /* db config */ - std::string GetDBConfigStrPath(); - std::string GetDBConfigStrSlavePath(); + std::string GetDBConfigStrPrimaryPath(); + std::string GetDBConfigStrSecondaryPath(); std::string GetDBConfigStrBackendUrl(); std::string GetDBConfigStrArchiveDiskThreshold(); std::string GetDBConfigStrArchiveDaysThreshold(); - std::string GetDBConfigStrBufferSize(); + std::string GetDBConfigStrInsertBufferSize(); std::string GetDBConfigStrBuildIndexGPU(); /* metric config */ - std::string GetMetricConfigStrAutoBootup(); + std::string GetMetricConfigStrEnableMonitor(); std::string GetMetricConfigStrCollector(); std::string GetMetricConfigStrPrometheusPort(); @@ -185,81 +188,81 @@ class Config { public: /* server config */ - Status GetServerConfigAddress(std::string& value); - Status GetServerConfigPort(std::string& value); - Status GetServerConfigMode(std::string& value); - Status GetServerConfigTimeZone(std::string& value); + Status GetServerConfigAddress(std::string &value); + Status GetServerConfigPort(std::string &value); + Status GetServerConfigDeployMode(std::string &value); + Status GetServerConfigTimeZone(std::string &value); /* db config */ - Status GetDBConfigPath(std::string& value); - Status GetDBConfigSlavePath(std::string& value); - Status GetDBConfigBackendUrl(std::string& value); - Status GetDBConfigArchiveDiskThreshold(int32_t& value); - Status GetDBConfigArchiveDaysThreshold(int32_t& value); - Status GetDBConfigBufferSize(int32_t& value); - Status GetDBConfigBuildIndexGPU(int32_t& value); + Status GetDBConfigPrimaryPath(std::string &value); + Status GetDBConfigSecondaryPath(std::string &value); + Status GetDBConfigBackendUrl(std::string &value); + Status GetDBConfigArchiveDiskThreshold(int32_t &value); + Status GetDBConfigArchiveDaysThreshold(int32_t &value); + Status GetDBConfigInsertBufferSize(int32_t &value); + Status GetDBConfigBuildIndexGPU(int32_t &value); /* metric config */ - Status GetMetricConfigAutoBootup(bool& value); - Status GetMetricConfigCollector(std::string& value); - Status GetMetricConfigPrometheusPort(std::string& value); + Status GetMetricConfigEnableMonitor(bool &value); + Status GetMetricConfigCollector(std::string &value); + Status GetMetricConfigPrometheusPort(std::string &value); /* cache config */ - Status GetCacheConfigCpuMemCapacity(int32_t& value); - Status GetCacheConfigCpuMemThreshold(float& value); - Status GetCacheConfigGpuMemCapacity(int32_t& value); - Status GetCacheConfigGpuMemThreshold(float& value); - Status GetCacheConfigCacheInsertData(bool& value); + Status GetCacheConfigCpuMemCapacity(int32_t &value); + Status GetCacheConfigCpuMemThreshold(float &value); + Status GetCacheConfigGpuMemCapacity(int32_t &value); + Status GetCacheConfigGpuMemThreshold(float &value); + Status GetCacheConfigCacheInsertData(bool &value); /* engine config */ - Status GetEngineConfigBlasThreshold(int32_t& value); - Status GetEngineConfigOmpThreadNum(int32_t& value); + Status GetEngineConfigBlasThreshold(int32_t &value); + Status GetEngineConfigOmpThreadNum(int32_t &value); /* resource config */ - Status GetResourceConfigMode(std::string& value); - Status GetResourceConfigPool(std::vector& value); + Status GetResourceConfigMode(std::string &value); + Status GetResourceConfigPool(std::vector &value); public: /* server config */ - Status SetServerConfigAddress(const std::string& value); - Status SetServerConfigPort(const std::string& value); - Status SetServerConfigMode(const std::string& value); - Status SetServerConfigTimeZone(const std::string& value); + Status SetServerConfigAddress(const std::string &value); + Status SetServerConfigPort(const std::string &value); + Status SetServerConfigDeployMode(const std::string &value); + Status SetServerConfigTimeZone(const std::string &value); /* db config */ - Status SetDBConfigPath(const std::string& value); - Status SetDBConfigSlavePath(const std::string& value); - Status SetDBConfigBackendUrl(const std::string& value); - Status SetDBConfigArchiveDiskThreshold(const std::string& value); - Status SetDBConfigArchiveDaysThreshold(const std::string& value); - Status SetDBConfigBufferSize(const std::string& value); - Status SetDBConfigBuildIndexGPU(const std::string& value); + Status SetDBConfigPrimaryPath(const std::string &value); + Status SetDBConfigSecondaryPath(const std::string &value); + Status SetDBConfigBackendUrl(const std::string &value); + Status SetDBConfigArchiveDiskThreshold(const std::string &value); + Status SetDBConfigArchiveDaysThreshold(const std::string &value); + Status SetDBConfigInsertBufferSize(const std::string &value); + Status SetDBConfigBuildIndexGPU(const std::string &value); /* metric config */ - Status SetMetricConfigAutoBootup(const std::string& value); - Status SetMetricConfigCollector(const std::string& value); - Status SetMetricConfigPrometheusPort(const std::string& value); + Status SetMetricConfigEnableMonitor(const std::string &value); + Status SetMetricConfigCollector(const std::string &value); + Status SetMetricConfigPrometheusPort(const std::string &value); /* cache config */ - Status SetCacheConfigCpuMemCapacity(const std::string& value); - Status SetCacheConfigCpuMemThreshold(const std::string& value); - Status SetCacheConfigGpuMemCapacity(const std::string& value); - Status SetCacheConfigGpuMemThreshold(const std::string& value); - Status SetCacheConfigCacheInsertData(const std::string& value); + Status SetCacheConfigCpuMemCapacity(const std::string &value); + Status SetCacheConfigCpuMemThreshold(const std::string &value); + Status SetCacheConfigGpuMemCapacity(const std::string &value); + Status SetCacheConfigGpuMemThreshold(const std::string &value); + Status SetCacheConfigCacheInsertData(const std::string &value); /* engine config */ - Status SetEngineConfigBlasThreshold(const std::string& value); - Status SetEngineConfigOmpThreadNum(const std::string& value); + Status SetEngineConfigBlasThreshold(const std::string &value); + Status SetEngineConfigOmpThreadNum(const std::string &value); /* resource config */ - Status SetResourceConfigMode(const std::string& value); + Status SetResourceConfigMode(const std::string &value); private: std::unordered_map> config_map_; std::mutex mutex_; }; -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 16b13a1249..66104bfe5a 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -16,13 +16,14 @@ // under the License. -#include "DBWrapper.h" +#include "server/DBWrapper.h" #include "Config.h" #include "db/DBFactory.h" #include "utils/CommonUtil.h" #include "utils/Log.h" #include "utils/StringHelpFunctions.h" +#include #include #include @@ -31,7 +32,6 @@ namespace milvus { namespace server { DBWrapper::DBWrapper() { - } Status DBWrapper::StartService() { @@ -44,13 +44,13 @@ Status DBWrapper::StartService() { if (!s.ok()) return s; std::string path; - s = config.GetDBConfigPath(path); + s = config.GetDBConfigPrimaryPath(path); if (!s.ok()) return s; opt.meta_.path_ = path + "/db"; std::string db_slave_path; - s = config.GetDBConfigSlavePath(db_slave_path); + s = config.GetDBConfigSecondaryPath(db_slave_path); if (!s.ok()) return s; StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_); @@ -60,20 +60,19 @@ Status DBWrapper::StartService() { if (!s.ok()) return s; std::string mode; - s = config.GetServerConfigMode(mode); + s = config.GetServerConfigDeployMode(mode); if (!s.ok()) return s; if (mode == "single") { opt.mode_ = engine::DBOptions::MODE::SINGLE; - } - else if (mode == "cluster") { - opt.mode_ = engine::DBOptions::MODE::CLUSTER; - } - else if (mode == "read_only") { - opt.mode_ = engine::DBOptions::MODE::READ_ONLY; - } - else { - std::cerr << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl; + } else if (mode == "cluster_readonly") { + opt.mode_ = engine::DBOptions::MODE::CLUSTER_READONLY; + } else if (mode == "cluster_writable") { + opt.mode_ = engine::DBOptions::MODE::CLUSTER_WRITABLE; + } else { + std::cerr << + "ERROR: mode specified in server_config must be ['single', 'cluster_readonly', 'cluster_writable']" + << std::endl; kill(0, SIGUSR1); } @@ -86,7 +85,7 @@ Status DBWrapper::StartService() { SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; } else { uint32_t sys_thread_cnt = 8; - if(CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { + if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { omp_thread = (int32_t)ceil(sys_thread_cnt*0.5); omp_set_num_threads(omp_thread); } @@ -116,14 +115,14 @@ Status DBWrapper::StartService() { //create db root folder Status status = CommonUtil::CreateDirectory(opt.meta_.path_); - if(!status.ok()) { + if (!status.ok()) { std::cerr << "ERROR! Failed to create database root path: " << opt.meta_.path_ << std::endl; kill(0, SIGUSR1); } - for(auto& path : opt.meta_.slave_paths_) { + for (auto& path : opt.meta_.slave_paths_) { status = CommonUtil::CreateDirectory(path); - if(!status.ok()) { + if (!status.ok()) { std::cerr << "ERROR! Failed to create database slave path: " << path << std::endl; kill(0, SIGUSR1); } @@ -143,13 +142,13 @@ Status DBWrapper::StartService() { } Status DBWrapper::StopService() { - if(db_) { + if (db_) { db_->Stop(); } return Status::OK(); } -} -} -} \ No newline at end of file +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/DBWrapper.h b/cpp/src/server/DBWrapper.h index 99b8f90500..4648f4b59d 100644 --- a/cpp/src/server/DBWrapper.h +++ b/cpp/src/server/DBWrapper.h @@ -27,12 +27,12 @@ namespace milvus { namespace server { class DBWrapper { -private: + private: DBWrapper(); ~DBWrapper() = default; -public: - static DBWrapper& GetInstance() { + public: + static DBWrapper &GetInstance() { static DBWrapper wrapper; return wrapper; } @@ -48,10 +48,10 @@ public: return db_; } -private: + private: engine::DBPtr db_; }; -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 57081845a5..f0675fc84d 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#include "server/Server.h" + #include #include #include @@ -24,7 +26,6 @@ #include #include -#include "Server.h" #include "server/grpc_impl/GrpcServer.h" #include "server/Config.h" #include "utils/Log.h" @@ -36,7 +37,6 @@ #include "wrapper/KnowhereResource.h" #include "DBWrapper.h" - namespace zilliz { namespace milvus { namespace server { @@ -116,7 +116,7 @@ Server::Daemonize() { } // Close all open fd - for (long fd = sysconf(_SC_OPEN_MAX); fd > 0; fd--) { + for (int64_t fd = sysconf(_SC_OPEN_MAX); fd > 0; fd--) { close(fd); } @@ -172,9 +172,9 @@ Server::Start() { time_zone = "CUT"; } else { int time_bias = std::stoi(time_zone.substr(3, std::string::npos)); - if (time_bias == 0) + if (time_bias == 0) { time_zone = "CUT"; - else if (time_bias > 0) { + } else if (time_bias > 0) { time_zone = "CUT" + std::to_string(-time_bias); } else { time_zone = "CUT+" + std::to_string(-time_bias); @@ -194,7 +194,6 @@ Server::Start() { StartService(); std::cout << "Milvus server start successfully." << std::endl; - } catch (std::exception &ex) { std::cerr << "Milvus server encounter exception: " << ex.what(); } @@ -232,15 +231,20 @@ Server::Stop() { std::cerr << "Milvus server is closed!" << std::endl; } - ErrorCode Server::LoadConfig() { - Config& config = Config::GetInstance(); + Config &config = Config::GetInstance(); Status s = config.LoadConfigFile(config_filename_); if (!s.ok()) { std::cerr << "Failed to load config file: " << config_filename_ << std::endl; exit(0); } + + s = config.ValidateConfig(); + if (!s.ok()) { + std::cerr << "Config check fail: " << s.message() << std::endl; + exit(0); + } return SERVER_SUCCESS; } @@ -260,6 +264,6 @@ Server::StopService() { engine::KnowhereResource::Finalize(); } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index 8507552b75..9bd0ce13b5 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -22,7 +22,6 @@ #include #include - namespace zilliz { namespace milvus { namespace server { @@ -58,6 +57,6 @@ class Server { std::string log_config_file_; }; // Server -} // server -} // sql -} // zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp index c85d1b2fcd..a64422aab7 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp @@ -16,10 +16,12 @@ // under the License. -#include "GrpcRequestHandler.h" -#include "GrpcRequestTask.h" +#include "server/grpc_impl/GrpcRequestHandler.h" +#include "server/grpc_impl/GrpcRequestTask.h" #include "utils/TimeRecorder.h" +#include + namespace zilliz { namespace milvus { namespace server { @@ -29,7 +31,6 @@ namespace grpc { GrpcRequestHandler::CreateTable(::grpc::ServerContext *context, const ::milvus::grpc::TableSchema *request, ::milvus::grpc::Status *response) { - BaseTaskPtr task_ptr = CreateTableTask::Create(request); GrpcRequestScheduler::ExecTask(task_ptr, response); return ::grpc::Status::OK; @@ -39,7 +40,6 @@ GrpcRequestHandler::CreateTable(::grpc::ServerContext *context, GrpcRequestHandler::HasTable(::grpc::ServerContext *context, const ::milvus::grpc::TableName *request, ::milvus::grpc::BoolReply *response) { - bool has_table = false; BaseTaskPtr task_ptr = HasTableTask::Create(request->table_name(), has_table); ::milvus::grpc::Status grpc_status; @@ -61,9 +61,8 @@ GrpcRequestHandler::DropTable(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::CreateIndex(::grpc::ServerContext *context, - const ::milvus::grpc::IndexParam *request, - ::milvus::grpc::Status *response) { - + const ::milvus::grpc::IndexParam *request, + ::milvus::grpc::Status *response) { BaseTaskPtr task_ptr = CreateIndexTask::Create(request); GrpcRequestScheduler::ExecTask(task_ptr, response); return ::grpc::Status::OK; @@ -71,9 +70,8 @@ GrpcRequestHandler::CreateIndex(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::Insert(::grpc::ServerContext *context, - const ::milvus::grpc::InsertParam *request, - ::milvus::grpc::VectorIds *response) { - + const ::milvus::grpc::InsertParam *request, + ::milvus::grpc::VectorIds *response) { BaseTaskPtr task_ptr = InsertTask::Create(request, response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -86,7 +84,6 @@ GrpcRequestHandler::Insert(::grpc::ServerContext *context, GrpcRequestHandler::Search(::grpc::ServerContext *context, const ::milvus::grpc::SearchParam *request, ::milvus::grpc::TopKQueryResultList *response) { - std::vector file_id_array; BaseTaskPtr task_ptr = SearchTask::Create(request, file_id_array, response); ::milvus::grpc::Status grpc_status; @@ -100,7 +97,6 @@ GrpcRequestHandler::Search(::grpc::ServerContext *context, GrpcRequestHandler::SearchInFiles(::grpc::ServerContext *context, const ::milvus::grpc::SearchInFilesParam *request, ::milvus::grpc::TopKQueryResultList *response) { - std::vector file_id_array; for (int i = 0; i < request->file_id_array_size(); i++) { file_id_array.push_back(request->file_id_array(i)); @@ -118,7 +114,6 @@ GrpcRequestHandler::SearchInFiles(::grpc::ServerContext *context, GrpcRequestHandler::DescribeTable(::grpc::ServerContext *context, const ::milvus::grpc::TableName *request, ::milvus::grpc::TableSchema *response) { - BaseTaskPtr task_ptr = DescribeTableTask::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -129,9 +124,8 @@ GrpcRequestHandler::DescribeTable(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::CountTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::TableRowCount *response) { - + const ::milvus::grpc::TableName *request, + ::milvus::grpc::TableRowCount *response) { int64_t row_count = 0; BaseTaskPtr task_ptr = CountTableTask::Create(request->table_name(), row_count); ::milvus::grpc::Status grpc_status; @@ -146,7 +140,6 @@ GrpcRequestHandler::CountTable(::grpc::ServerContext *context, GrpcRequestHandler::ShowTables(::grpc::ServerContext *context, const ::milvus::grpc::Command *request, ::milvus::grpc::TableNameList *response) { - BaseTaskPtr task_ptr = ShowTablesTask::Create(response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -157,9 +150,8 @@ GrpcRequestHandler::ShowTables(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::Cmd(::grpc::ServerContext *context, - const ::milvus::grpc::Command *request, - ::milvus::grpc::StringReply *response) { - + const ::milvus::grpc::Command *request, + ::milvus::grpc::StringReply *response) { std::string result; BaseTaskPtr task_ptr = CmdTask::Create(request->cmd(), result); ::milvus::grpc::Status grpc_status; @@ -172,8 +164,8 @@ GrpcRequestHandler::Cmd(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::DeleteByRange(::grpc::ServerContext *context, - const ::milvus::grpc::DeleteByRangeParam *request, - ::milvus::grpc::Status *response) { + const ::milvus::grpc::DeleteByRangeParam *request, + ::milvus::grpc::Status *response) { BaseTaskPtr task_ptr = DeleteByRangeTask::Create(request); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -184,8 +176,8 @@ GrpcRequestHandler::DeleteByRange(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::PreloadTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) { + const ::milvus::grpc::TableName *request, + ::milvus::grpc::Status *response) { BaseTaskPtr task_ptr = PreloadTableTask::Create(request->table_name()); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -196,8 +188,8 @@ GrpcRequestHandler::PreloadTable(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::DescribeIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::IndexParam *response) { + const ::milvus::grpc::TableName *request, + ::milvus::grpc::IndexParam *response) { BaseTaskPtr task_ptr = DescribeIndexTask::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -208,8 +200,8 @@ GrpcRequestHandler::DescribeIndex(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::DropIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) { + const ::milvus::grpc::TableName *request, + ::milvus::grpc::Status *response) { BaseTaskPtr task_ptr = DropIndexTask::Create(request->table_name()); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -218,8 +210,7 @@ GrpcRequestHandler::DropIndex(::grpc::ServerContext *context, return ::grpc::Status::OK; } - -} -} -} -} \ No newline at end of file +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.h b/cpp/src/server/grpc_impl/GrpcRequestHandler.h index a8a70eb88d..549e1d9d35 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.h @@ -20,15 +20,15 @@ #include #include -#include "milvus.grpc.pb.h" -#include "status.pb.h" +#include "grpc/gen-milvus/milvus.grpc.pb.h" +#include "grpc/gen-status/status.pb.h" namespace zilliz { namespace milvus { namespace server { namespace grpc { class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { -public: + public: /** * @brief Create table method * @@ -103,8 +103,7 @@ public: */ ::grpc::Status CreateIndex(::grpc::ServerContext *context, - const ::milvus::grpc::IndexParam *request, ::milvus::grpc::Status *response) override; - + const ::milvus::grpc::IndexParam *request, ::milvus::grpc::Status *response) override; /** * @brief Insert vector array to table @@ -123,8 +122,8 @@ public: */ ::grpc::Status Insert(::grpc::ServerContext *context, - const ::milvus::grpc::InsertParam *request, - ::milvus::grpc::VectorIds *response) override; + const ::milvus::grpc::InsertParam *request, + ::milvus::grpc::VectorIds *response) override; /** * @brief Query vector @@ -213,8 +212,8 @@ public: */ ::grpc::Status CountTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::TableRowCount *response) override; + const ::milvus::grpc::TableName *request, + ::milvus::grpc::TableRowCount *response) override; /** * @brief List all tables in database @@ -253,8 +252,8 @@ public: */ ::grpc::Status Cmd(::grpc::ServerContext *context, - const ::milvus::grpc::Command *request, - ::milvus::grpc::StringReply *response) override; + const ::milvus::grpc::Command *request, + ::milvus::grpc::StringReply *response) override; /** * @brief delete table by range @@ -291,8 +290,8 @@ public: */ ::grpc::Status PreloadTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) override; + const ::milvus::grpc::TableName *request, + ::milvus::grpc::Status *response) override; /** * @brief Describe index @@ -310,8 +309,8 @@ public: */ ::grpc::Status DescribeIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::IndexParam *response) override; + const ::milvus::grpc::TableName *request, + ::milvus::grpc::IndexParam *response) override; /** * @brief Drop index @@ -329,12 +328,12 @@ public: */ ::grpc::Status DropIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) override; - + const ::milvus::grpc::TableName *request, + ::milvus::grpc::Status *response) override; }; -} -} -} -} + +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp index fb8daa3540..4c58195f37 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp @@ -15,101 +15,107 @@ // specific language governing permissions and limitations // under the License. -#include "GrpcRequestScheduler.h" +#include "server/grpc_impl/GrpcRequestScheduler.h" #include "utils/Log.h" -#include "src/grpc/gen-status/status.pb.h" +#include "grpc/gen-status/status.pb.h" + +#include namespace zilliz { namespace milvus { namespace server { namespace grpc { -using namespace ::milvus; - namespace { - ::milvus::grpc::ErrorCode ErrorMap(ErrorCode code) { - static const std::map code_map = { - {SERVER_UNEXPECTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, - {SERVER_UNSUPPORTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, - {SERVER_NULL_POINTER, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, - {SERVER_INVALID_ARGUMENT, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_FILE_NOT_FOUND, ::milvus::grpc::ErrorCode::FILE_NOT_FOUND}, - {SERVER_NOT_IMPLEMENT, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, - {SERVER_BLOCKING_QUEUE_EMPTY, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, - {SERVER_CANNOT_CREATE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FOLDER}, - {SERVER_CANNOT_CREATE_FILE, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FILE}, - {SERVER_CANNOT_DELETE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_DELETE_FOLDER}, - {SERVER_CANNOT_DELETE_FILE, ::milvus::grpc::ErrorCode::CANNOT_DELETE_FILE}, - {SERVER_TABLE_NOT_EXIST, ::milvus::grpc::ErrorCode::TABLE_NOT_EXISTS}, - {SERVER_INVALID_TABLE_NAME, ::milvus::grpc::ErrorCode::ILLEGAL_TABLE_NAME}, - {SERVER_INVALID_TABLE_DIMENSION, ::milvus::grpc::ErrorCode::ILLEGAL_DIMENSION}, - {SERVER_INVALID_TIME_RANGE, ::milvus::grpc::ErrorCode::ILLEGAL_RANGE}, - {SERVER_INVALID_VECTOR_DIMENSION, ::milvus::grpc::ErrorCode::ILLEGAL_DIMENSION}, +::milvus::grpc::ErrorCode +ErrorMap(ErrorCode code) { + static const std::map code_map = { + {SERVER_UNEXPECTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_UNSUPPORTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_NULL_POINTER, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_INVALID_ARGUMENT, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, + {SERVER_FILE_NOT_FOUND, ::milvus::grpc::ErrorCode::FILE_NOT_FOUND}, + {SERVER_NOT_IMPLEMENT, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_BLOCKING_QUEUE_EMPTY, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_CANNOT_CREATE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FOLDER}, + {SERVER_CANNOT_CREATE_FILE, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FILE}, + {SERVER_CANNOT_DELETE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_DELETE_FOLDER}, + {SERVER_CANNOT_DELETE_FILE, ::milvus::grpc::ErrorCode::CANNOT_DELETE_FILE}, + {SERVER_TABLE_NOT_EXIST, ::milvus::grpc::ErrorCode::TABLE_NOT_EXISTS}, + {SERVER_INVALID_TABLE_NAME, ::milvus::grpc::ErrorCode::ILLEGAL_TABLE_NAME}, + {SERVER_INVALID_TABLE_DIMENSION, ::milvus::grpc::ErrorCode::ILLEGAL_DIMENSION}, + {SERVER_INVALID_TIME_RANGE, ::milvus::grpc::ErrorCode::ILLEGAL_RANGE}, + {SERVER_INVALID_VECTOR_DIMENSION, ::milvus::grpc::ErrorCode::ILLEGAL_DIMENSION}, - {SERVER_INVALID_INDEX_TYPE, ::milvus::grpc::ErrorCode::ILLEGAL_INDEX_TYPE}, - {SERVER_INVALID_ROWRECORD, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD}, - {SERVER_INVALID_ROWRECORD_ARRAY, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD}, - {SERVER_INVALID_TOPK, ::milvus::grpc::ErrorCode::ILLEGAL_TOPK}, - {SERVER_INVALID_NPROBE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_INVALID_INDEX_NLIST, ::milvus::grpc::ErrorCode::ILLEGAL_NLIST}, - {SERVER_INVALID_INDEX_METRIC_TYPE,::milvus::grpc::ErrorCode::ILLEGAL_METRIC_TYPE}, - {SERVER_INVALID_INDEX_FILE_SIZE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_ILLEGAL_VECTOR_ID, ::milvus::grpc::ErrorCode::ILLEGAL_VECTOR_ID}, - {SERVER_ILLEGAL_SEARCH_RESULT, ::milvus::grpc::ErrorCode::ILLEGAL_SEARCH_RESULT}, - {SERVER_CACHE_ERROR, ::milvus::grpc::ErrorCode::CACHE_FAILED}, - {DB_META_TRANSACTION_FAILED, ::milvus::grpc::ErrorCode::META_FAILED}, - {SERVER_BUILD_INDEX_ERROR, ::milvus::grpc::ErrorCode::BUILD_INDEX_ERROR}, - {SERVER_OUT_OF_MEMORY, ::milvus::grpc::ErrorCode::OUT_OF_MEMORY}, - }; + {SERVER_INVALID_INDEX_TYPE, ::milvus::grpc::ErrorCode::ILLEGAL_INDEX_TYPE}, + {SERVER_INVALID_ROWRECORD, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD}, + {SERVER_INVALID_ROWRECORD_ARRAY, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD}, + {SERVER_INVALID_TOPK, ::milvus::grpc::ErrorCode::ILLEGAL_TOPK}, + {SERVER_INVALID_NPROBE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, + {SERVER_INVALID_INDEX_NLIST, ::milvus::grpc::ErrorCode::ILLEGAL_NLIST}, + {SERVER_INVALID_INDEX_METRIC_TYPE, ::milvus::grpc::ErrorCode::ILLEGAL_METRIC_TYPE}, + {SERVER_INVALID_INDEX_FILE_SIZE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, + {SERVER_ILLEGAL_VECTOR_ID, ::milvus::grpc::ErrorCode::ILLEGAL_VECTOR_ID}, + {SERVER_ILLEGAL_SEARCH_RESULT, ::milvus::grpc::ErrorCode::ILLEGAL_SEARCH_RESULT}, + {SERVER_CACHE_ERROR, ::milvus::grpc::ErrorCode::CACHE_FAILED}, + {DB_META_TRANSACTION_FAILED, ::milvus::grpc::ErrorCode::META_FAILED}, + {SERVER_BUILD_INDEX_ERROR, ::milvus::grpc::ErrorCode::BUILD_INDEX_ERROR}, + {SERVER_OUT_OF_MEMORY, ::milvus::grpc::ErrorCode::OUT_OF_MEMORY}, + }; - if(code_map.find(code) != code_map.end()) { - return code_map.at(code); - } else { - return ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR; - } + if (code_map.find(code) != code_map.end()) { + return code_map.at(code); + } else { + return ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR; } } +} // namespace //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// GrpcBaseTask::GrpcBaseTask(const std::string &task_group, bool async) - : task_group_(task_group), - async_(async), - done_(false) { - + : task_group_(task_group), + async_(async), + done_(false) { } GrpcBaseTask::~GrpcBaseTask() { WaitToFinish(); } -Status GrpcBaseTask::Execute() { +Status +GrpcBaseTask::Execute() { status_ = OnExecute(); Done(); return status_; } -void GrpcBaseTask::Done() { +void +GrpcBaseTask::Done() { done_ = true; finish_cond_.notify_all(); } -Status GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string &error_msg) { +Status +GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string &error_msg) { status_ = Status(error_code, error_msg); SERVER_LOG_ERROR << error_msg; return status_; } -Status GrpcBaseTask::WaitToFinish() { +Status +GrpcBaseTask::WaitToFinish() { std::unique_lock lock(finish_mtx_); - finish_cond_.wait(lock, [this] { return done_; }); + finish_cond_.wait(lock, [this] { + return done_; + }); return status_; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// GrpcRequestScheduler::GrpcRequestScheduler() - : stopped_(false) { + : stopped_(false) { Start(); } @@ -117,7 +123,8 @@ GrpcRequestScheduler::~GrpcRequestScheduler() { Stop(); } -void GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status) { +void +GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status) { if (task_ptr == nullptr) { return; } @@ -127,7 +134,7 @@ void GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Statu if (!task_ptr->IsAsync()) { task_ptr->WaitToFinish(); - const Status& status = task_ptr->status(); + const Status &status = task_ptr->status(); if (!status.ok()) { grpc_status->set_reason(status.message()); grpc_status->set_error_code(ErrorMap(status.code())); @@ -135,7 +142,8 @@ void GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Statu } } -void GrpcRequestScheduler::Start() { +void +GrpcRequestScheduler::Start() { if (!stopped_) { return; } @@ -143,7 +151,8 @@ void GrpcRequestScheduler::Start() { stopped_ = false; } -void GrpcRequestScheduler::Stop() { +void +GrpcRequestScheduler::Stop() { if (stopped_) { return; } @@ -168,7 +177,8 @@ void GrpcRequestScheduler::Stop() { SERVER_LOG_INFO << "Scheduler stopped"; } -Status GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) { +Status +GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) { if (task_ptr == nullptr) { return Status::OK(); } @@ -186,8 +196,8 @@ Status GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) { return task_ptr->WaitToFinish();//sync execution } - -void GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { +void +GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { if (task_queue == nullptr) { return; } @@ -210,7 +220,8 @@ void GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { } } -Status GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { +Status +GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { std::lock_guard lock(queue_mtx_); std::string group_name = task_ptr->TaskGroup(); @@ -230,7 +241,7 @@ Status GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { return Status::OK(); } -} -} -} -} +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h index e1217540fc..df5357a4bb 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h @@ -19,12 +19,14 @@ #include "utils/Status.h" #include "utils/BlockingQueue.h" -#include "status.grpc.pb.h" -#include "status.pb.h" +#include "grpc/gen-status/status.grpc.pb.h" +#include "grpc/gen-status/status.pb.h" #include #include #include +#include +#include namespace zilliz { namespace milvus { @@ -32,30 +34,36 @@ namespace server { namespace grpc { class GrpcBaseTask { -protected: - GrpcBaseTask(const std::string &task_group, bool async = false); + protected: + explicit GrpcBaseTask(const std::string &task_group, bool async = false); virtual ~GrpcBaseTask(); -public: + public: Status Execute(); void Done(); Status WaitToFinish(); - std::string TaskGroup() const { return task_group_; } + std::string TaskGroup() const { + return task_group_; + } - const Status& status() const { return status_; } + const Status &status() const { + return status_; + } - bool IsAsync() const { return async_; } + bool IsAsync() const { + return async_; + } -protected: + protected: virtual Status OnExecute() = 0; Status SetStatus(ErrorCode error_code, const std::string &msg); -protected: + protected: mutable std::mutex finish_mtx_; std::condition_variable finish_cond_; @@ -71,7 +79,7 @@ using TaskQueuePtr = std::shared_ptr; using ThreadPtr = std::shared_ptr; class GrpcRequestScheduler { -public: + public: static GrpcRequestScheduler &GetInstance() { static GrpcRequestScheduler scheduler; return scheduler; @@ -85,7 +93,7 @@ public: static void ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status); -protected: + protected: GrpcRequestScheduler(); virtual ~GrpcRequestScheduler(); @@ -94,7 +102,7 @@ protected: Status PutTaskToQueue(const BaseTaskPtr &task_ptr); -private: + private: mutable std::mutex queue_mtx_; std::map task_groups_; @@ -104,7 +112,7 @@ private: bool stopped_; }; -} -} -} -} +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index ca6b77e960..5702b80bec 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -15,22 +15,24 @@ // specific language governing permissions and limitations // under the License. -#include +#include "server/grpc_impl/GrpcRequestTask.h" -#include "GrpcRequestTask.h" +#include +#include +#include +#include +//#include + +#include "server/Server.h" +#include "server/DBWrapper.h" #include "utils/CommonUtil.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" #include "utils/ValidationUtil.h" -#include "../DBWrapper.h" -#include "version.h" #include "GrpcServer.h" #include "db/Utils.h" #include "scheduler/SchedInst.h" -//#include - -#include "server/Server.h" - +#include "../../../version.h" namespace zilliz { namespace milvus { @@ -45,86 +47,87 @@ using DB_META = zilliz::milvus::engine::meta::Meta; using DB_DATE = zilliz::milvus::engine::meta::DateT; namespace { - engine::EngineType EngineType(int type) { - static std::map map_type = { - {0, engine::EngineType::INVALID}, - {1, engine::EngineType::FAISS_IDMAP}, - {2, engine::EngineType::FAISS_IVFFLAT}, - {3, engine::EngineType::FAISS_IVFSQ8}, - }; +engine::EngineType +EngineType(int type) { + static std::map map_type = { + {0, engine::EngineType::INVALID}, + {1, engine::EngineType::FAISS_IDMAP}, + {2, engine::EngineType::FAISS_IVFFLAT}, + {3, engine::EngineType::FAISS_IVFSQ8}, + }; - if (map_type.find(type) == map_type.end()) { - return engine::EngineType::INVALID; - } - - return map_type[type]; + if (map_type.find(type) == map_type.end()) { + return engine::EngineType::INVALID; } - int IndexType(engine::EngineType type) { - static std::map map_type = { - {engine::EngineType::INVALID, 0}, - {engine::EngineType::FAISS_IDMAP, 1}, - {engine::EngineType::FAISS_IVFFLAT, 2}, - {engine::EngineType::FAISS_IVFSQ8, 3}, - }; - - if (map_type.find(type) == map_type.end()) { - return 0; - } - - return map_type[type]; - } - - constexpr long DAY_SECONDS = 24 * 60 * 60; - - Status - ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range> &range_array, - std::vector &dates) { - dates.clear(); - for (auto &range : range_array) { - time_t tt_start, tt_end; - tm tm_start, tm_end; - if (!CommonUtil::TimeStrToTime(range.start_value(), tt_start, tm_start)) { - return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); - } - - if (!CommonUtil::TimeStrToTime(range.end_value(), tt_end, tm_end)) { - return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); - } - - long days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) / - DAY_SECONDS; - if (days == 0) { - return Status(SERVER_INVALID_TIME_RANGE, - "Invalid time range: " + range.start_value() + " to " + range.end_value()); - } - - //range: [start_day, end_day) - for (long i = 0; i < days; i++) { - time_t tt_day = tt_start + DAY_SECONDS * i; - tm tm_day; - CommonUtil::ConvertTime(tt_day, tm_day); - - long date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + - tm_day.tm_mday;//according to db logic - dates.push_back(date); - } - } - - return Status::OK(); - } + return map_type[type]; } +int +IndexType(engine::EngineType type) { + static std::map map_type = { + {engine::EngineType::INVALID, 0}, + {engine::EngineType::FAISS_IDMAP, 1}, + {engine::EngineType::FAISS_IVFFLAT, 2}, + {engine::EngineType::FAISS_IVFSQ8, 3}, + }; + + if (map_type.find(type) == map_type.end()) { + return 0; + } + + return map_type[type]; +} + +constexpr int64_t DAY_SECONDS = 24 * 60 * 60; + +Status +ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range> &range_array, + std::vector &dates) { + dates.clear(); + for (auto &range : range_array) { + time_t tt_start, tt_end; + tm tm_start, tm_end; + if (!CommonUtil::TimeStrToTime(range.start_value(), tt_start, tm_start)) { + return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); + } + + if (!CommonUtil::TimeStrToTime(range.end_value(), tt_end, tm_end)) { + return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); + } + + int64_t days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) / + DAY_SECONDS; + if (days == 0) { + return Status(SERVER_INVALID_TIME_RANGE, + "Invalid time range: " + range.start_value() + " to " + range.end_value()); + } + + //range: [start_day, end_day) + for (int64_t i = 0; i < days; i++) { + time_t tt_day = tt_start + DAY_SECONDS * i; + tm tm_day; + CommonUtil::ConvertTime(tt_day, tm_day); + + int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + + tm_day.tm_mday;//according to db logic + dates.push_back(date); + } + } + + return Status::OK(); +} +} // namespace + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// CreateTableTask::CreateTableTask(const ::milvus::grpc::TableSchema *schema) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - schema_(schema) { - + : GrpcBaseTask(DDL_DML_TASK_GROUP), + schema_(schema) { } BaseTaskPtr CreateTableTask::Create(const ::milvus::grpc::TableSchema *schema) { - if(schema == nullptr) { + if (schema == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; } @@ -168,12 +171,11 @@ CreateTableTask::OnExecute() { status = DBWrapper::DB()->CreateTable(table_info); if (!status.ok()) { //table could exist - if(status.code() == DB_ALREADY_EXIST) { + if (status.code() == DB_ALREADY_EXIST) { return Status(SERVER_INVALID_TABLE_NAME, status.message()); } return status; } - } catch (std::exception &ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -185,9 +187,9 @@ CreateTableTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// DescribeTableTask::DescribeTableTask(const std::string &table_name, ::milvus::grpc::TableSchema *schema) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name), - schema_(schema) { + : GrpcBaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name), + schema_(schema) { } BaseTaskPtr @@ -218,7 +220,6 @@ DescribeTableTask::OnExecute() { schema_->set_dimension(table_info.dimension_); schema_->set_index_file_size(table_info.index_file_size_); schema_->set_metric_type(table_info.metric_type_); - } catch (std::exception &ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -230,13 +231,13 @@ DescribeTableTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// CreateIndexTask::CreateIndexTask(const ::milvus::grpc::IndexParam *index_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - index_param_(index_param) { + : GrpcBaseTask(DDL_DML_TASK_GROUP), + index_param_(index_param) { } BaseTaskPtr CreateIndexTask::Create(const ::milvus::grpc::IndexParam *index_param) { - if(index_param == nullptr) { + if (index_param == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; } @@ -295,10 +296,9 @@ CreateIndexTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// HasTableTask::HasTableTask(const std::string &table_name, bool &has_table) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name), - has_table_(has_table) { - + : GrpcBaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name), + has_table_(has_table) { } BaseTaskPtr @@ -333,9 +333,8 @@ HasTableTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// DropTableTask::DropTableTask(const std::string &table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name) { - + : GrpcBaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name) { } BaseTaskPtr @@ -385,9 +384,8 @@ DropTableTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ShowTablesTask::ShowTablesTask(::milvus::grpc::TableNameList *table_name_list) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_list_(table_name_list) { - + : GrpcBaseTask(DDL_DML_TASK_GROUP), + table_name_list_(table_name_list) { } BaseTaskPtr @@ -419,8 +417,8 @@ InsertTask::InsertTask(const ::milvus::grpc::InsertParam *insert_param, BaseTaskPtr InsertTask::Create(const ::milvus::grpc::InsertParam *insert_param, - ::milvus::grpc::VectorIds *record_ids) { - if(insert_param == nullptr) { + ::milvus::grpc::VectorIds *record_ids) { + if (insert_param == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; } @@ -444,7 +442,7 @@ InsertTask::OnExecute() { if (!record_ids_->vector_id_array().empty()) { if (record_ids_->vector_id_array().size() != insert_param_->row_record_array_size()) { return Status(SERVER_ILLEGAL_VECTOR_ID, - "Size of vector ids is not equal to row record array size"); + "Size of vector ids is not equal to row record array size"); } } @@ -455,7 +453,7 @@ InsertTask::OnExecute() { if (!status.ok()) { if (status.code() == DB_NOT_FOUND) { return Status(SERVER_TABLE_NOT_EXIST, - "Table " + insert_param_->table_name() + " not exists"); + "Table " + insert_param_->table_name() + " not exists"); } else { return status; } @@ -465,19 +463,22 @@ InsertTask::OnExecute() { //all user provide id, or all internal id bool user_provide_ids = !insert_param_->row_id_array().empty(); //user already provided id before, all insert action require user id - if((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) && !user_provide_ids) { - return Status(SERVER_ILLEGAL_VECTOR_ID, "Table vector ids are user defined, please provide id for this batch"); + if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) && !user_provide_ids) { + return Status(SERVER_ILLEGAL_VECTOR_ID, + "Table vector ids are user defined, please provide id for this batch"); } //user didn't provided id before, no need to provide user id - if((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) && user_provide_ids) { - return Status(SERVER_ILLEGAL_VECTOR_ID, "Table vector ids are auto generated, no need to provide id for this batch"); + if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) && user_provide_ids) { + return Status(SERVER_ILLEGAL_VECTOR_ID, + "Table vector ids are auto generated, no need to provide id for this batch"); } rc.RecordSection("check validation"); #ifdef MILVUS_ENABLE_PROFILING - std::string fname = "/tmp/insert_" + std::to_string(this->insert_param_->row_record_array_size()) + ".profiling"; + std::string fname = "/tmp/insert_" + std::to_string(this->insert_param_->row_record_array_size()) + + ".profiling"; ProfilerStart(fname.c_str()); #endif @@ -493,8 +494,8 @@ InsertTask::OnExecute() { if (vec_dim != table_info.dimension_) { ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; std::string error_msg = "Invalid row record dimension: " + std::to_string(vec_dim) - + " vs. table dimension:" + - std::to_string(table_info.dimension_); + + " vs. table dimension:" + + std::to_string(table_info.dimension_); return Status(error_code, error_msg); } memcpy(&vec_f[i * table_info.dimension_], @@ -507,10 +508,10 @@ InsertTask::OnExecute() { //step 5: insert vectors auto vec_count = (uint64_t) insert_param_->row_record_array_size(); std::vector vec_ids(insert_param_->row_id_array_size(), 0); - if(!insert_param_->row_id_array().empty()) { - const int64_t* src_data = insert_param_->row_id_array().data(); - int64_t* target_data = vec_ids.data(); - memcpy(target_data, src_data, (size_t)(sizeof(int64_t)*insert_param_->row_id_array_size())); + if (!insert_param_->row_id_array().empty()) { + const int64_t *src_data = insert_param_->row_id_array().data(); + int64_t *target_data = vec_ids.data(); + memcpy(target_data, src_data, (size_t) (sizeof(int64_t) * insert_param_->row_id_array_size())); } status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), vec_count, vec_f.data(), vec_ids); @@ -525,13 +526,13 @@ InsertTask::OnExecute() { auto ids_size = record_ids_->vector_id_array_size(); if (ids_size != vec_count) { std::string msg = "Add " + std::to_string(vec_count) + " vectors but only return " - + std::to_string(ids_size) + " id"; + + std::to_string(ids_size) + " id"; return Status(SERVER_ILLEGAL_VECTOR_ID, msg); } //step 6: update table flag user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID - : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; + : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; status = DBWrapper::DB()->UpdateTableFlag(insert_param_->table_name(), table_info.flag_); #ifdef MILVUS_ENABLE_PROFILING @@ -540,7 +541,6 @@ InsertTask::OnExecute() { rc.RecordSection("add vectors to engine"); rc.ElapseFromBegin("total cost"); - } catch (std::exception &ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -556,19 +556,17 @@ SearchTask::SearchTask(const ::milvus::grpc::SearchParam *search_vector_infos, search_param_(search_vector_infos), file_id_array_(file_id_array), topk_result_list(response) { - } BaseTaskPtr SearchTask::Create(const ::milvus::grpc::SearchParam *search_vector_infos, const std::vector &file_id_array, ::milvus::grpc::TopKQueryResultList *response) { - if(search_vector_infos == nullptr) { + if (search_vector_infos == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; } - return std::shared_ptr(new SearchTask(search_vector_infos, file_id_array, - response)); + return std::shared_ptr(new SearchTask(search_vector_infos, file_id_array, response)); } Status @@ -640,7 +638,7 @@ SearchTask::OnExecute() { if (query_vec_dim != table_info.dimension_) { ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; std::string error_msg = "Invalid row record dimension: " + std::to_string(query_vec_dim) - + " vs. table dimension:" + std::to_string(table_info.dimension_); + + " vs. table dimension:" + std::to_string(table_info.dimension_); return Status(error_code, error_msg); } @@ -655,16 +653,17 @@ SearchTask::OnExecute() { auto record_count = (uint64_t) search_param_->query_record_array().size(); #ifdef MILVUS_ENABLE_PROFILING - std::string fname = "/tmp/search_nq_" + std::to_string(this->search_param_->query_record_array_size()) + ".profiling"; + std::string fname = "/tmp/search_nq_" + std::to_string(this->search_param_->query_record_array_size()) + + ".profiling"; ProfilerStart(fname.c_str()); #endif if (file_id_array_.empty()) { status = DBWrapper::DB()->Query(table_name_, (size_t) top_k, record_count, nprobe, - vec_f.data(), dates, results); + vec_f.data(), dates, results); } else { status = DBWrapper::DB()->Query(table_name_, file_id_array_, (size_t) top_k, - record_count, nprobe, vec_f.data(), dates, results); + record_count, nprobe, vec_f.data(), dates, results); } #ifdef MILVUS_ENABLE_PROFILING @@ -682,7 +681,7 @@ SearchTask::OnExecute() { if (results.size() != record_count) { std::string msg = "Search " + std::to_string(record_count) + " vectors but only return " - + std::to_string(results.size()) + " results"; + + std::to_string(results.size()) + " results"; return Status(SERVER_ILLEGAL_SEARCH_RESULT, msg); } @@ -699,8 +698,6 @@ SearchTask::OnExecute() { //step 8: print time cost percent rc.RecordSection("construct result and send"); rc.ElapseFromBegin("totally cost"); - - } catch (std::exception &ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -710,10 +707,9 @@ SearchTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// CountTableTask::CountTableTask(const std::string &table_name, int64_t &row_count) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name), - row_count_(row_count) { - + : GrpcBaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name), + row_count_(row_count) { } BaseTaskPtr @@ -742,7 +738,6 @@ CountTableTask::OnExecute() { row_count_ = (int64_t) row_count; rc.ElapseFromBegin("total cost"); - } catch (std::exception &ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -752,10 +747,9 @@ CountTableTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// CmdTask::CmdTask(const std::string &cmd, std::string &result) - : GrpcBaseTask(PING_TASK_GROUP), - cmd_(cmd), - result_(result) { - + : GrpcBaseTask(PING_TASK_GROUP), + cmd_(cmd), + result_(result) { } BaseTaskPtr @@ -769,8 +763,7 @@ CmdTask::OnExecute() { result_ = MILVUS_VERSION; } else if (cmd_ == "tasktable") { result_ = scheduler::ResMgrInst::GetInstance()->DumpTaskTables(); - } - else { + } else { result_ = "OK"; } @@ -779,16 +772,17 @@ CmdTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// DeleteByRangeTask::DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - delete_by_range_param_(delete_by_range_param){ + : GrpcBaseTask(DDL_DML_TASK_GROUP), + delete_by_range_param_(delete_by_range_param) { } BaseTaskPtr DeleteByRangeTask::Create(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param) { - if(delete_by_range_param == nullptr) { + if (delete_by_range_param == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; } + return std::shared_ptr(new DeleteByRangeTask(delete_by_range_param)); } @@ -838,23 +832,21 @@ DeleteByRangeTask::OnExecute() { if (!status.ok()) { return status; } - } catch (std::exception &ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// PreloadTableTask::PreloadTableTask(const std::string &table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name) { - + : GrpcBaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name) { } BaseTaskPtr -PreloadTableTask::Create(const std::string &table_name){ +PreloadTableTask::Create(const std::string &table_name) { return std::shared_ptr(new PreloadTableTask(table_name)); } @@ -889,12 +881,11 @@ DescribeIndexTask::DescribeIndexTask(const std::string &table_name, : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name), index_param_(index_param) { - } BaseTaskPtr DescribeIndexTask::Create(const std::string &table_name, - ::milvus::grpc::IndexParam *index_param){ + ::milvus::grpc::IndexParam *index_param) { return std::shared_ptr(new DescribeIndexTask(table_name, index_param)); } @@ -932,11 +923,10 @@ DescribeIndexTask::OnExecute() { DropIndexTask::DropIndexTask(const std::string &table_name) : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name) { - } BaseTaskPtr -DropIndexTask::Create(const std::string &table_name){ +DropIndexTask::Create(const std::string &table_name) { return std::shared_ptr(new DropIndexTask(table_name)); } @@ -975,7 +965,7 @@ DropIndexTask::OnExecute() { return Status::OK(); } -} -} -} -} +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.h b/cpp/src/server/grpc_impl/GrpcRequestTask.h index a0c4540cb3..4c8c038d44 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.h +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.h @@ -16,15 +16,18 @@ // under the License. #pragma once -#include "GrpcRequestScheduler.h" + +#include "server/grpc_impl/GrpcRequestScheduler.h" #include "utils/Status.h" #include "db/Types.h" -#include "milvus.grpc.pb.h" -#include "status.pb.h" +#include "grpc/gen-milvus/milvus.grpc.pb.h" +#include "grpc/gen-status/status.pb.h" #include #include +#include +#include namespace zilliz { namespace milvus { @@ -33,138 +36,130 @@ namespace grpc { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CreateTableTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const ::milvus::grpc::TableSchema *schema); -protected: - explicit - CreateTableTask(const ::milvus::grpc::TableSchema *request); + protected: + explicit CreateTableTask(const ::milvus::grpc::TableSchema *request); Status OnExecute() override; -private: + private: const ::milvus::grpc::TableSchema *schema_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class HasTableTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &table_name, bool &has_table); -protected: + protected: HasTableTask(const std::string &request, bool &has_table); Status OnExecute() override; - -private: + private: std::string table_name_; bool &has_table_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DescribeTableTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &table_name, ::milvus::grpc::TableSchema *schema); -protected: + protected: DescribeTableTask(const std::string &table_name, ::milvus::grpc::TableSchema *schema); Status OnExecute() override; - -private: + private: std::string table_name_; ::milvus::grpc::TableSchema *schema_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DropTableTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &table_name); -protected: - explicit - DropTableTask(const std::string &table_name); + protected: + explicit DropTableTask(const std::string &table_name); Status OnExecute() override; - -private: + private: std::string table_name_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CreateIndexTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const ::milvus::grpc::IndexParam *index_Param); -protected: - explicit - CreateIndexTask(const ::milvus::grpc::IndexParam *index_Param); + protected: + explicit CreateIndexTask(const ::milvus::grpc::IndexParam *index_Param); Status OnExecute() override; - -private: + private: const ::milvus::grpc::IndexParam *index_param_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class ShowTablesTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(::milvus::grpc::TableNameList *table_name_list); -protected: - explicit - ShowTablesTask(::milvus::grpc::TableNameList *table_name_list); + protected: + explicit ShowTablesTask(::milvus::grpc::TableNameList *table_name_list); Status OnExecute() override; -private: + private: ::milvus::grpc::TableNameList *table_name_list_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class InsertTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const ::milvus::grpc::InsertParam *insert_Param, ::milvus::grpc::VectorIds *record_ids_); -protected: + protected: InsertTask(const ::milvus::grpc::InsertParam *insert_Param, - ::milvus::grpc::VectorIds *record_ids_); + ::milvus::grpc::VectorIds *record_ids_); Status OnExecute() override; -private: + private: const ::milvus::grpc::InsertParam *insert_param_; ::milvus::grpc::VectorIds *record_ids_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class SearchTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const ::milvus::grpc::SearchParam *search_param, const std::vector &file_id_array, ::milvus::grpc::TopKQueryResultList *response); -protected: + protected: SearchTask(const ::milvus::grpc::SearchParam *search_param, const std::vector &file_id_array, ::milvus::grpc::TopKQueryResultList *response); @@ -172,7 +167,7 @@ protected: Status OnExecute() override; -private: + private: const ::milvus::grpc::SearchParam *search_param_; std::vector file_id_array_; ::milvus::grpc::TopKQueryResultList *topk_result_list; @@ -180,107 +175,106 @@ private: //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CountTableTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &table_name, int64_t &row_count); -protected: + protected: CountTableTask(const std::string &table_name, int64_t &row_count); Status OnExecute() override; -private: + private: std::string table_name_; int64_t &row_count_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CmdTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &cmd, std::string &result); -protected: + protected: CmdTask(const std::string &cmd, std::string &result); Status OnExecute() override; -private: + private: std::string cmd_; std::string &result_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DeleteByRangeTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param); -protected: - DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param); + protected: + explicit DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param); Status OnExecute() override; -private: + private: const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class PreloadTableTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &table_name); -protected: - PreloadTableTask(const std::string &table_name); + protected: + explicit PreloadTableTask(const std::string &table_name); Status OnExecute() override; -private: + private: std::string table_name_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DescribeIndexTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &table_name, - ::milvus::grpc::IndexParam *index_param); + ::milvus::grpc::IndexParam *index_param); -protected: + protected: DescribeIndexTask(const std::string &table_name, - ::milvus::grpc::IndexParam *index_param); + ::milvus::grpc::IndexParam *index_param); Status OnExecute() override; -private: + private: std::string table_name_; ::milvus::grpc::IndexParam *index_param_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DropIndexTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &table_name); -protected: - DropIndexTask(const std::string &table_name); + protected: + explicit DropIndexTask(const std::string &table_name); Status OnExecute() override; -private: + private: std::string table_name_; - }; -} -} -} -} \ No newline at end of file +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcServer.cpp b/cpp/src/server/grpc_impl/GrpcServer.cpp index 7e20921549..065271dd55 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcServer.cpp @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include "milvus.grpc.pb.h" -#include "GrpcServer.h" +#include "grpc/gen-milvus/milvus.grpc.pb.h" +#include "server/grpc_impl/GrpcServer.h" #include "server/Config.h" #include "server/DBWrapper.h" #include "utils/Log.h" @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -35,14 +36,12 @@ #include #include - namespace zilliz { namespace milvus { namespace server { namespace grpc { - -constexpr long MESSAGE_SIZE = -1; +constexpr int64_t MESSAGE_SIZE = -1; //this class is to check port occupation during server start class NoReusePortOption : public ::grpc::ServerBuilderOption { @@ -52,11 +51,9 @@ class NoReusePortOption : public ::grpc::ServerBuilderOption { } void UpdatePlugins(std::vector> *plugins) override { - } }; - void GrpcServer::Start() { thread_ptr_ = std::make_shared(&GrpcServer::StartService, this); @@ -117,7 +114,7 @@ GrpcServer::StopService() { return Status::OK(); } -} -} -} -} \ No newline at end of file +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcServer.h b/cpp/src/server/grpc_impl/GrpcServer.h index a861692fac..9101f144b3 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.h +++ b/cpp/src/server/grpc_impl/GrpcServer.h @@ -19,12 +19,12 @@ #include "utils/Status.h" +#include #include #include #include #include - namespace zilliz { namespace milvus { namespace server { @@ -52,7 +52,7 @@ class GrpcServer { std::shared_ptr thread_ptr_; }; -} -} -} -} +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/BlockingQueue.h b/cpp/src/utils/BlockingQueue.h index 6e46ed1bb1..b98fe28ef8 100644 --- a/cpp/src/utils/BlockingQueue.h +++ b/cpp/src/utils/BlockingQueue.h @@ -29,8 +29,9 @@ namespace server { template class BlockingQueue { -public: - BlockingQueue() : mtx(), full_(), empty_() {} + public: + BlockingQueue() : mtx(), full_(), empty_() { + } BlockingQueue(const BlockingQueue &rhs) = delete; @@ -50,7 +51,7 @@ public: void SetCapacity(const size_t capacity); -private: + private: mutable std::mutex mtx; std::condition_variable full_; std::condition_variable empty_; @@ -58,9 +59,8 @@ private: size_t capacity_ = 32; }; -} -} -} - +} // namespace server +} // namespace milvus +} // namespace zilliz #include "./BlockingQueue.inl" diff --git a/cpp/src/utils/BlockingQueue.inl b/cpp/src/utils/BlockingQueue.inl index 86237f3322..ed15aac77a 100644 --- a/cpp/src/utils/BlockingQueue.inl +++ b/cpp/src/utils/BlockingQueue.inl @@ -18,7 +18,6 @@ #pragma once - namespace zilliz { namespace milvus { namespace server { @@ -26,8 +25,10 @@ namespace server { template void BlockingQueue::Put(const T &task) { - std::unique_lock lock(mtx); - full_.wait(lock, [this] { return (queue_.size() < capacity_); }); + std::unique_lock lock(mtx); + full_.wait(lock, [this] { + return (queue_.size() < capacity_); + }); queue_.push(task); empty_.notify_all(); @@ -36,8 +37,10 @@ BlockingQueue::Put(const T &task) { template T BlockingQueue::Take() { - std::unique_lock lock(mtx); - empty_.wait(lock, [this] { return !queue_.empty(); }); + std::unique_lock lock(mtx); + empty_.wait(lock, [this] { + return !queue_.empty(); + }); T front(queue_.front()); queue_.pop(); @@ -48,15 +51,17 @@ BlockingQueue::Take() { template size_t BlockingQueue::Size() { - std::lock_guard lock(mtx); + std::lock_guard lock(mtx); return queue_.size(); } template T BlockingQueue::Front() { - std::unique_lock lock(mtx); - empty_.wait(lock, [this] { return !queue_.empty(); }); + std::unique_lock lock(mtx); + empty_.wait(lock, [this] { + return !queue_.empty(); + }); T front(queue_.front()); return front; @@ -65,8 +70,10 @@ BlockingQueue::Front() { template T BlockingQueue::Back() { - std::unique_lock lock(mtx); - empty_.wait(lock, [this] { return !queue_.empty(); }); + std::unique_lock lock(mtx); + empty_.wait(lock, [this] { + return !queue_.empty(); + }); T back(queue_.back()); return back; @@ -75,7 +82,7 @@ BlockingQueue::Back() { template bool BlockingQueue::Empty() { - std::unique_lock lock(mtx); + std::unique_lock lock(mtx); return queue_.empty(); } @@ -85,7 +92,7 @@ BlockingQueue::SetCapacity(const size_t capacity) { capacity_ = (capacity > 0 ? capacity : capacity_); } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index ff985dcd7d..0116e321e0 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "CommonUtil.h" +#include "utils/CommonUtil.h" #include "utils/Log.h" #include @@ -44,7 +44,8 @@ namespace server { namespace fs = boost::filesystem; -bool CommonUtil::GetSystemMemInfo(unsigned long &total_mem, unsigned long &free_mem) { +bool +CommonUtil::GetSystemMemInfo(uint64_t &total_mem, uint64_t &free_mem) { struct sysinfo info; int ret = sysinfo(&info); total_mem = info.totalram; @@ -53,7 +54,8 @@ bool CommonUtil::GetSystemMemInfo(unsigned long &total_mem, unsigned long &free_ return ret == 0;//succeed 0, failed -1 } -bool CommonUtil::GetSystemAvailableThreads(unsigned int &thread_count) { +bool +CommonUtil::GetSystemAvailableThreads(uint32_t &thread_count) { //threadCnt = std::thread::hardware_concurrency(); thread_count = sysconf(_SC_NPROCESSORS_CONF); thread_count *= THREAD_MULTIPLY_CPU; @@ -63,7 +65,8 @@ bool CommonUtil::GetSystemAvailableThreads(unsigned int &thread_count) { return true; } -bool CommonUtil::IsDirectoryExist(const std::string &path) { +bool +CommonUtil::IsDirectoryExist(const std::string &path) { DIR *dp = nullptr; if ((dp = opendir(path.c_str())) == nullptr) { return false; @@ -73,8 +76,9 @@ bool CommonUtil::IsDirectoryExist(const std::string &path) { return true; } -Status CommonUtil::CreateDirectory(const std::string &path) { - if(path.empty()) { +Status +CommonUtil::CreateDirectory(const std::string &path) { + if (path.empty()) { return Status::OK(); } @@ -87,7 +91,7 @@ Status CommonUtil::CreateDirectory(const std::string &path) { fs::path fs_path(path); fs::path parent_path = fs_path.parent_path(); Status err_status = CreateDirectory(parent_path.string()); - if(!err_status.ok()){ + if (!err_status.ok()) { return err_status; } @@ -96,7 +100,7 @@ Status CommonUtil::CreateDirectory(const std::string &path) { return Status::OK();//already exist } - int makeOK = mkdir(path.c_str(), S_IRWXU|S_IRGRP|S_IROTH); + int makeOK = mkdir(path.c_str(), S_IRWXU | S_IRGRP | S_IROTH); if (makeOK != 0) { return Status(SERVER_UNEXPECTED_ERROR, "failed to create directory: " + path); } @@ -105,37 +109,38 @@ Status CommonUtil::CreateDirectory(const std::string &path) { } namespace { - void RemoveDirectory(const std::string &path) { - DIR *dir = nullptr; - struct dirent *dmsg; - char file_name[256]; - char folder_name[256]; +void +RemoveDirectory(const std::string &path) { + DIR *dir = nullptr; + struct dirent *dmsg; + const int32_t buf_size = 256; + char file_name[buf_size]; - strcpy(folder_name, path.c_str()); - strcat(folder_name, "/%s"); - if ((dir = opendir(path.c_str())) != nullptr) { - while ((dmsg = readdir(dir)) != nullptr) { - if (strcmp(dmsg->d_name, ".") != 0 - && strcmp(dmsg->d_name, "..") != 0) { - sprintf(file_name, folder_name, dmsg->d_name); - std::string tmp = file_name; - if (tmp.find(".") == std::string::npos) { - RemoveDirectory(file_name); - } - remove(file_name); + std::string folder_name = path + "/%s"; + if ((dir = opendir(path.c_str())) != nullptr) { + while ((dmsg = readdir(dir)) != nullptr) { + if (strcmp(dmsg->d_name, ".") != 0 + && strcmp(dmsg->d_name, "..") != 0) { + snprintf(file_name, buf_size, folder_name.c_str(), dmsg->d_name); + std::string tmp = file_name; + if (tmp.find(".") == std::string::npos) { + RemoveDirectory(file_name); } + remove(file_name); } } - - if (dir != nullptr) { - closedir(dir); - } - remove(path.c_str()); } -} -Status CommonUtil::DeleteDirectory(const std::string &path) { - if(path.empty()) { + if (dir != nullptr) { + closedir(dir); + } + remove(path.c_str()); +} +} // namespace + +Status +CommonUtil::DeleteDirectory(const std::string &path) { + if (path.empty()) { return Status::OK(); } @@ -149,58 +154,63 @@ Status CommonUtil::DeleteDirectory(const std::string &path) { return Status::OK(); } -bool CommonUtil::IsFileExist(const std::string &path) { +bool +CommonUtil::IsFileExist(const std::string &path) { return (access(path.c_str(), F_OK) == 0); } -uint64_t CommonUtil::GetFileSize(const std::string &path) { +uint64_t +CommonUtil::GetFileSize(const std::string &path) { struct stat file_info; if (stat(path.c_str(), &file_info) < 0) { return 0; } else { - return (uint64_t)file_info.st_size; + return (uint64_t) file_info.st_size; } } -std::string CommonUtil::GetFileName(std::string filename) { +std::string +CommonUtil::GetFileName(std::string filename) { int pos = filename.find_last_of('/'); return filename.substr(pos + 1); } -std::string CommonUtil::GetExePath() { +std::string +CommonUtil::GetExePath() { const size_t buf_len = 1024; char buf[buf_len]; size_t cnt = readlink("/proc/self/exe", buf, buf_len); - if(cnt < 0|| cnt >= buf_len) { + if (cnt < 0 || cnt >= buf_len) { return ""; } buf[cnt] = '\0'; std::string exe_path = buf; - if(exe_path.rfind('/') != exe_path.length()){ + if (exe_path.rfind('/') != exe_path.length()) { std::string sub_str = exe_path.substr(0, exe_path.rfind('/')); return sub_str + "/"; } return exe_path; } -bool CommonUtil::TimeStrToTime(const std::string& time_str, - time_t &time_integer, - tm &time_struct, - const std::string& format) { +bool +CommonUtil::TimeStrToTime(const std::string &time_str, + time_t &time_integer, + tm &time_struct, + const std::string &format) { time_integer = 0; memset(&time_struct, 0, sizeof(tm)); int ret = sscanf(time_str.c_str(), - format.c_str(), - &(time_struct.tm_year), - &(time_struct.tm_mon), - &(time_struct.tm_mday), - &(time_struct.tm_hour), - &(time_struct.tm_min), - &(time_struct.tm_sec)); - if(ret <= 0) { + format.c_str(), + &(time_struct.tm_year), + &(time_struct.tm_mon), + &(time_struct.tm_mday), + &(time_struct.tm_hour), + &(time_struct.tm_min), + &(time_struct.tm_sec)); + if (ret <= 0) { return false; } @@ -211,15 +221,16 @@ bool CommonUtil::TimeStrToTime(const std::string& time_str, return true; } -void CommonUtil::ConvertTime(time_t time_integer, tm &time_struct) { - tm* t_m = localtime (&time_integer); - memcpy(&time_struct, t_m, sizeof(tm)); +void +CommonUtil::ConvertTime(time_t time_integer, tm &time_struct) { + localtime_r(&time_integer, &time_struct); } -void CommonUtil::ConvertTime(tm time_struct, time_t &time_integer) { +void +CommonUtil::ConvertTime(tm time_struct, time_t &time_integer) { time_integer = mktime(&time_struct); } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/CommonUtil.h b/cpp/src/utils/CommonUtil.h index 7e1349bf9e..b059067d50 100755 --- a/cpp/src/utils/CommonUtil.h +++ b/cpp/src/utils/CommonUtil.h @@ -22,15 +22,14 @@ #include #include - namespace zilliz { namespace milvus { namespace server { class CommonUtil { public: - static bool GetSystemMemInfo(unsigned long &total_mem, unsigned long &free_mem); - static bool GetSystemAvailableThreads(unsigned int &thread_count); + static bool GetSystemMemInfo(uint64_t &total_mem, uint64_t &free_mem); + static bool GetSystemAvailableThreads(uint32_t &thread_count); static bool IsFileExist(const std::string &path); static uint64_t GetFileSize(const std::string &path); @@ -41,16 +40,16 @@ class CommonUtil { static std::string GetFileName(std::string filename); static std::string GetExePath(); - static bool TimeStrToTime(const std::string& time_str, - time_t &time_integer, - tm &time_struct, - const std::string& format = "%d-%d-%d %d:%d:%d"); + static bool TimeStrToTime(const std::string &time_str, + time_t &time_integer, + tm &time_struct, + const std::string &format = "%d-%d-%d %d:%d:%d"); static void ConvertTime(time_t time_integer, tm &time_struct); static void ConvertTime(tm time_struct, time_t &time_integer); }; -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index e903dcb2fb..d0bcc5cf8d 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -28,6 +28,7 @@ using ErrorCode = int32_t; constexpr ErrorCode SERVER_SUCCESS = 0; constexpr ErrorCode SERVER_ERROR_CODE_BASE = 0x30000; + constexpr ErrorCode ToServerErrorCode(const ErrorCode error_code) { return SERVER_ERROR_CODE_BASE + error_code; @@ -35,6 +36,7 @@ ToServerErrorCode(const ErrorCode error_code) { constexpr ErrorCode DB_SUCCESS = 0; constexpr ErrorCode DB_ERROR_CODE_BASE = 0x40000; + constexpr ErrorCode ToDbErrorCode(const ErrorCode error_code) { return DB_ERROR_CODE_BASE + error_code; @@ -42,6 +44,7 @@ ToDbErrorCode(const ErrorCode error_code) { constexpr ErrorCode KNOWHERE_SUCCESS = 0; constexpr ErrorCode KNOWHERE_ERROR_CODE_BASE = 0x50000; + constexpr ErrorCode ToKnowhereErrorCode(const ErrorCode error_code) { return KNOWHERE_ERROR_CODE_BASE + error_code; @@ -96,26 +99,27 @@ constexpr ErrorCode KNOWHERE_UNEXPECTED_ERROR = ToKnowhereErrorCode(3); constexpr ErrorCode KNOWHERE_NO_SPACE = ToKnowhereErrorCode(4); namespace server { - class ServerException : public std::exception { - public: - ServerException(ErrorCode error_code, - const std::string &message = std::string()) - : error_code_(error_code), message_(message) {} +class ServerException : public std::exception { + public: + ServerException(ErrorCode error_code, + const std::string &message = std::string()) + : error_code_(error_code), message_(message) { + } - public: - ErrorCode error_code() const { - return error_code_; - } + public: + ErrorCode error_code() const { + return error_code_; + } - virtual const char *what() const noexcept { - return message_.c_str(); - } + virtual const char *what() const noexcept { + return message_.c_str(); + } - private: - ErrorCode error_code_; - std::string message_; - }; -} + private: + ErrorCode error_code_; + std::string message_; +}; +} // namespace server } // namespace milvus } // namespace zilliz diff --git a/cpp/src/utils/Exception.h b/cpp/src/utils/Exception.h index 3a9ab89294..7e30c372bc 100644 --- a/cpp/src/utils/Exception.h +++ b/cpp/src/utils/Exception.h @@ -26,17 +26,17 @@ namespace zilliz { namespace milvus { class Exception : public std::exception { -public: - Exception(ErrorCode code, const std::string& message) + public: + Exception(ErrorCode code, const std::string &message) : code_(code), message_(message) { - } + } ErrorCode code() const throw() { return code_; } - virtual const char* what() const throw() { + virtual const char *what() const throw() { if (message_.empty()) { return "Default Exception."; } else { @@ -44,24 +44,23 @@ public: } } - virtual ~Exception() throw() {}; + virtual ~Exception() throw() { + } -protected: + protected: ErrorCode code_; std::string message_; }; class InvalidArgumentException : public Exception { -public: + public: InvalidArgumentException() : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") { + } - }; - InvalidArgumentException(const std::string& message) + explicit InvalidArgumentException(const std::string &message) : Exception(SERVER_INVALID_ARGUMENT, message) { - - }; - + } }; } // namespace milvus diff --git a/cpp/src/utils/Log.h b/cpp/src/utils/Log.h index 2610a3a8bf..b1402d9e3e 100644 --- a/cpp/src/utils/Log.h +++ b/cpp/src/utils/Log.h @@ -17,7 +17,7 @@ #pragma once -#include "easylogging++.h" +#include "utils/easylogging++.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index 3a3fe0c7e2..0e710a6e3a 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. +#include "utils/LogUtil.h" + #include #include #include -#include "LogUtil.h" - namespace zilliz { namespace milvus { namespace server { @@ -35,7 +35,8 @@ static int fatal_idx = 0; } // TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename -void RolloutHandler(const char *filename, std::size_t size, el::Level level) { +void +RolloutHandler(const char *filename, std::size_t size, el::Level level) { char *dirc = strdup(filename); char *basec = strdup(filename); char *dir = dirname(dirc); @@ -80,7 +81,8 @@ void RolloutHandler(const char *filename, std::size_t size, el::Level level) { } } -Status InitLog(const std::string &log_config_file) { +Status +InitLog(const std::string &log_config_file) { el::Configurations conf(log_config_file); el::Loggers::reconfigureAllLoggers(conf); @@ -91,7 +93,6 @@ Status InitLog(const std::string &log_config_file) { return Status::OK(); } - -} // server -} // milvus -} // zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/LogUtil.h b/cpp/src/utils/LogUtil.h index d86b301d9f..e3f7bed51f 100644 --- a/cpp/src/utils/LogUtil.h +++ b/cpp/src/utils/LogUtil.h @@ -18,18 +18,20 @@ #pragma once #include "utils/Status.h" +#include "utils/easylogging++.h" #include #include -#include "easylogging++.h" namespace zilliz { namespace milvus { namespace server { -Status InitLog(const std::string& log_config_file); +Status +InitLog(const std::string &log_config_file); -void RolloutHandler(const char *filename, std::size_t size, el::Level level); +void +RolloutHandler(const char *filename, std::size_t size, el::Level level); #define SHOW_LOCATION #ifdef SHOW_LOCATION @@ -38,6 +40,6 @@ void RolloutHandler(const char *filename, std::size_t size, el::Level level); #define LOCATION_INFO "" #endif -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index 92678cfe6b..5f74852995 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -15,20 +15,20 @@ // specific language governing permissions and limitations // under the License. -#include "SignalUtil.h" +#include "utils/SignalUtil.h" #include "src/server/Server.h" #include "utils/Log.h" +#include #include #include - namespace zilliz { namespace milvus { namespace server { -void SignalUtil::HandleSignal(int signum) { - +void +SignalUtil::HandleSignal(int signum) { switch (signum) { case SIGINT: case SIGUSR2: { @@ -51,7 +51,8 @@ void SignalUtil::HandleSignal(int signum) { } } -void SignalUtil::PrintStacktrace() { +void +SignalUtil::PrintStacktrace() { SERVER_LOG_INFO << "Call stack:"; const int size = 32; @@ -65,6 +66,6 @@ void SignalUtil::PrintStacktrace() { free(stacktrace); } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/SignalUtil.h b/cpp/src/utils/SignalUtil.h index 39645cf989..c9cd41839b 100644 --- a/cpp/src/utils/SignalUtil.h +++ b/cpp/src/utils/SignalUtil.h @@ -27,6 +27,6 @@ class SignalUtil { static void PrintStacktrace(); }; -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/Status.cpp b/cpp/src/utils/Status.cpp index e21a279a4c..5b512b3369 100644 --- a/cpp/src/utils/Status.cpp +++ b/cpp/src/utils/Status.cpp @@ -14,7 +14,8 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -#include "Status.h" + +#include "utils/Status.h" #include @@ -23,12 +24,12 @@ namespace milvus { constexpr int CODE_WIDTH = sizeof(StatusCode); -Status::Status(StatusCode code, const std::string& msg) { +Status::Status(StatusCode code, const std::string &msg) { //4 bytes store code //4 bytes store message length //the left bytes store message string - const uint32_t length = (uint32_t)msg.size(); - char* result = new char[length + sizeof(length) + CODE_WIDTH]; + const uint32_t length = (uint32_t) msg.size(); + char *result = new char[length + sizeof(length) + CODE_WIDTH]; std::memcpy(result, &code, CODE_WIDTH); std::memcpy(result + CODE_WIDTH, &length, sizeof(length)); memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length); @@ -38,7 +39,6 @@ Status::Status(StatusCode code, const std::string& msg) { Status::Status() : state_(nullptr) { - } Status::~Status() { @@ -50,7 +50,7 @@ Status::Status(const Status &s) CopyFrom(s); } -Status& +Status & Status::operator=(const Status &s) { CopyFrom(s); return *this; @@ -61,7 +61,7 @@ Status::Status(Status &&s) MoveFrom(s); } -Status& +Status & Status::operator=(Status &&s) { MoveFrom(s); return *this; @@ -71,7 +71,7 @@ void Status::CopyFrom(const Status &s) { delete state_; state_ = nullptr; - if(s.state_ == nullptr) { + if (s.state_ == nullptr) { return; } @@ -79,7 +79,7 @@ Status::CopyFrom(const Status &s) { memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); int buff_len = length + sizeof(length) + CODE_WIDTH; state_ = new char[buff_len]; - memcpy((void*)state_, (void*)s.state_, buff_len); + memcpy((void *) state_, (void *) s.state_, buff_len); } void @@ -98,7 +98,7 @@ Status::message() const { std::string msg; uint32_t length = 0; memcpy(&length, state_ + CODE_WIDTH, sizeof(length)); - if(length > 0) { + if (length > 0) { msg.append(state_ + sizeof(length) + CODE_WIDTH, length); } @@ -113,26 +113,19 @@ Status::ToString() const { std::string result; switch (code()) { - case DB_SUCCESS: - result = "OK "; + case DB_SUCCESS:result = "OK "; break; - case DB_ERROR: - result = "Error: "; + case DB_ERROR:result = "Error: "; break; - case DB_META_TRANSACTION_FAILED: - result = "Database error: "; + case DB_META_TRANSACTION_FAILED:result = "Database error: "; break; - case DB_NOT_FOUND: - result = "Not found: "; + case DB_NOT_FOUND:result = "Not found: "; break; - case DB_ALREADY_EXIST: - result = "Already exist: "; + case DB_ALREADY_EXIST:result = "Already exist: "; break; - case DB_INVALID_PATH: - result = "Invalid path: "; + case DB_INVALID_PATH:result = "Invalid path: "; break; - default: - result = "Error code(" + std::to_string(code()) + "): "; + default:result = "Error code(" + std::to_string(code()) + "): "; break; } diff --git a/cpp/src/utils/Status.h b/cpp/src/utils/Status.h index fe06c8029d..8f8f238979 100644 --- a/cpp/src/utils/Status.h +++ b/cpp/src/utils/Status.h @@ -44,14 +44,18 @@ class Status { operator=(Status &&s); static Status - OK() { return Status(); } + OK() { + return Status(); + } bool - ok() const { return state_ == nullptr || code() == 0; } + ok() const { + return state_ == nullptr || code() == 0; + } StatusCode code() const { - return (state_ == nullptr) ? 0 : *(StatusCode*)(state_); + return (state_ == nullptr) ? 0 : *(StatusCode *) (state_); } std::string @@ -60,14 +64,14 @@ class Status { std::string ToString() const; -private: + private: inline void CopyFrom(const Status &s); inline void MoveFrom(Status &s); -private: + private: const char *state_ = nullptr; }; // Status diff --git a/cpp/src/utils/StringHelpFunctions.cpp b/cpp/src/utils/StringHelpFunctions.cpp index 068c376ea0..8c9e888d3a 100644 --- a/cpp/src/utils/StringHelpFunctions.cpp +++ b/cpp/src/utils/StringHelpFunctions.cpp @@ -15,13 +15,16 @@ // specific language governing permissions and limitations // under the License. -#include "StringHelpFunctions.h" +#include "utils/StringHelpFunctions.h" + +#include namespace zilliz { namespace milvus { namespace server { -void StringHelpFunctions::TrimStringBlank(std::string &string) { +void +StringHelpFunctions::TrimStringBlank(std::string &string) { if (!string.empty()) { static std::string s_format(" \n\r\t"); string.erase(0, string.find_first_not_of(s_format)); @@ -29,17 +32,19 @@ void StringHelpFunctions::TrimStringBlank(std::string &string) { } } -void StringHelpFunctions::TrimStringQuote(std::string &string, const std::string &qoute) { +void +StringHelpFunctions::TrimStringQuote(std::string &string, const std::string &qoute) { if (!string.empty()) { string.erase(0, string.find_first_not_of(qoute)); string.erase(string.find_last_not_of(qoute) + 1); } } -Status StringHelpFunctions::SplitStringByDelimeter(const std::string &str, - const std::string &delimeter, - std::vector &result) { - if(str.empty()) { +Status +StringHelpFunctions::SplitStringByDelimeter(const std::string &str, + const std::string &delimeter, + std::vector &result) { + if (str.empty()) { return Status::OK(); } @@ -58,10 +63,11 @@ Status StringHelpFunctions::SplitStringByDelimeter(const std::string &str, return Status::OK(); } -Status StringHelpFunctions::SplitStringByQuote(const std::string &str, - const std::string &delimeter, - const std::string "e, - std::vector &result) { +Status +StringHelpFunctions::SplitStringByQuote(const std::string &str, + const std::string &delimeter, + const std::string "e, + std::vector &result) { if (quote.empty()) { return SplitStringByDelimeter(str, delimeter, result); } @@ -120,6 +126,6 @@ Status StringHelpFunctions::SplitStringByQuote(const std::string &str, return Status::OK(); } -} -} -} \ No newline at end of file +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/StringHelpFunctions.h b/cpp/src/utils/StringHelpFunctions.h index d50b62d8a3..ebffa075ad 100644 --- a/cpp/src/utils/StringHelpFunctions.h +++ b/cpp/src/utils/StringHelpFunctions.h @@ -20,16 +20,17 @@ #include "utils/Status.h" #include +#include namespace zilliz { namespace milvus { namespace server { class StringHelpFunctions { -private: + private: StringHelpFunctions() = default; -public: + public: static void TrimStringBlank(std::string &string); static void TrimStringQuote(std::string &string, const std::string &qoute); @@ -56,9 +57,8 @@ public: const std::string &delimeter, const std::string "e, std::vector &result); - }; -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/ThreadPool.h b/cpp/src/utils/ThreadPool.h index 8060ee4cf6..0524f6206d 100644 --- a/cpp/src/utils/ThreadPool.h +++ b/cpp/src/utils/ThreadPool.h @@ -26,7 +26,7 @@ #include #include #include - +#include #define MAX_THREADS_NUM 32 @@ -34,8 +34,8 @@ namespace zilliz { namespace milvus { class ThreadPool { -public: - ThreadPool(size_t threads, size_t queue_size = 1000); + public: + explicit ThreadPool(size_t threads, size_t queue_size = 1000); template auto enqueue(F &&f, Args &&... args) @@ -43,7 +43,7 @@ public: ~ThreadPool(); -private: + private: // need to keep track of threads so we can join them std::vector workers_; @@ -60,53 +60,57 @@ private: bool stop; }; - // the constructor just launches some amount of workers inline ThreadPool::ThreadPool(size_t threads, size_t queue_size) - : max_queue_size_(queue_size), stop(false) { + : max_queue_size_(queue_size), stop(false) { for (size_t i = 0; i < threads; ++i) workers_.emplace_back( - [this] { - for (;;) { - std::function task; + [this] { + for (;;) { + std::function task; - { - std::unique_lock lock(this->queue_mutex_); - this->condition_.wait(lock, - [this] { return this->stop || !this->tasks_.empty(); }); - if (this->stop && this->tasks_.empty()) - return; - task = std::move(this->tasks_.front()); - this->tasks_.pop(); - } - this->condition_.notify_all(); - - task(); + { + std::unique_lock lock(this->queue_mutex_); + this->condition_.wait(lock, + [this] { + return this->stop || !this->tasks_.empty(); + }); + if (this->stop && this->tasks_.empty()) + return; + task = std::move(this->tasks_.front()); + this->tasks_.pop(); } + this->condition_.notify_all(); + + task(); } - ); + }); } // add new work item to the pool template -auto ThreadPool::enqueue(F &&f, Args &&... args) +auto +ThreadPool::enqueue(F &&f, Args &&... args) -> std::future::type> { using return_type = typename std::result_of::type; auto task = std::make_shared >( - std::bind(std::forward(f), std::forward(args)...) - ); + std::bind(std::forward(f), std::forward(args)...)); std::future res = task->get_future(); { std::unique_lock lock(queue_mutex_); this->condition_.wait(lock, - [this] { return this->tasks_.size() < max_queue_size_; }); + [this] { + return this->tasks_.size() < max_queue_size_; + }); // don't allow enqueueing after stopping the pool if (stop) throw std::runtime_error("enqueue on stopped ThreadPool"); - tasks_.emplace([task]() { (*task)(); }); + tasks_.emplace([task]() { + (*task)(); + }); } condition_.notify_all(); return res; @@ -119,10 +123,11 @@ inline ThreadPool::~ThreadPool() { stop = true; } condition_.notify_all(); - for (std::thread &worker: workers_) + for (std::thread &worker : workers_) { worker.join(); + } } -} -} +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/TimeRecorder.cpp b/cpp/src/utils/TimeRecorder.cpp index 6e4b5c1ab6..5246b35f13 100644 --- a/cpp/src/utils/TimeRecorder.cpp +++ b/cpp/src/utils/TimeRecorder.cpp @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include "TimeRecorder.h" +#include "utils/TimeRecorder.h" #include "utils/Log.h" - namespace zilliz { namespace milvus { @@ -100,5 +99,5 @@ TimeRecorder::ElapseFromBegin(const std::string &msg) { return span; } -} -} +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/TimeRecorder.h b/cpp/src/utils/TimeRecorder.h index c6efbaeb8a..2bb937e71f 100644 --- a/cpp/src/utils/TimeRecorder.h +++ b/cpp/src/utils/TimeRecorder.h @@ -20,14 +20,13 @@ #include #include - namespace zilliz { namespace milvus { class TimeRecorder { using stdclock = std::chrono::high_resolution_clock; -public: + public: TimeRecorder(const std::string &header, int64_t log_level = 1); @@ -39,15 +38,15 @@ public: static std::string GetTimeSpanStr(double span); -private: + private: void PrintTimeRecord(const std::string &msg, double span); -private: + private: std::string header_; stdclock::time_point start_; stdclock::time_point last_; int64_t log_level_; }; -} -} +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index c9ec2d046b..a6d83af1dc 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -16,16 +16,16 @@ // under the License. +#include "utils/ValidationUtil.h" #include "db/engine/ExecutionEngine.h" -#include "ValidationUtil.h" #include "Log.h" +#include #include #include #include #include - namespace zilliz { namespace milvus { namespace server { @@ -36,7 +36,6 @@ constexpr int32_t INDEX_FILE_SIZE_LIMIT = 4096; //index trigger size max = 4096 Status ValidationUtil::ValidateTableName(const std::string &table_name) { - // Table name shouldn't be empty. if (table_name.empty()) { std::string msg = "Empty table name"; @@ -78,8 +77,7 @@ ValidationUtil::ValidateTableDimension(int64_t dimension) { std::string msg = "Table dimension excceed the limitation: " + std::to_string(TABLE_DIMENSION_LIMIT); SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_VECTOR_DIMENSION, msg); - } - else { + } else { return Status::OK(); } } @@ -185,7 +183,6 @@ ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t &memory) { Status ValidationUtil::ValidateIpAddress(const std::string &ip_address) { - struct in_addr address; int result = inet_pton(AF_INET, ip_address.c_str(), &address); @@ -212,7 +209,7 @@ ValidationUtil::ValidateStringIsNumber(const std::string &str) { } try { int32_t value = std::stoi(str); - } catch(...) { + } catch (...) { return Status(SERVER_INVALID_ARGUMENT, "Invalid number"); } return Status::OK(); @@ -226,8 +223,7 @@ ValidationUtil::ValidateStringIsBool(const std::string &str) { s == "false" || s == "off" || s == "no" || s == "0" || s.empty()) { return Status::OK(); - } - else { + } else { return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str); } } @@ -236,7 +232,7 @@ Status ValidationUtil::ValidateStringIsFloat(const std::string &str) { try { float val = std::stof(str); - } catch(...) { + } catch (...) { return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str); } return Status::OK(); @@ -289,8 +285,7 @@ ValidationUtil::ValidateDbURI(const std::string &uri) { okay = false; } } - } - else { + } else { SERVER_LOG_ERROR << "Wrong URI format: URI = " << uri; okay = false; } @@ -298,6 +293,6 @@ ValidationUtil::ValidateDbURI(const std::string &uri) { return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid db backend uri")); } -} -} -} \ No newline at end of file +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index 789da0de40..44d6065a64 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -21,15 +21,17 @@ #include "db/meta/MetaTypes.h" #include "utils/Status.h" +#include + namespace zilliz { namespace milvus { namespace server { class ValidationUtil { -private: + private: ValidationUtil() = default; -public: + public: static Status ValidateTableName(const std::string &table_name); @@ -49,10 +51,10 @@ public: ValidateTableIndexMetricType(int32_t metric_type); static Status - ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema); + ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema &table_schema); static Status - ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema); + ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema &table_schema); static Status ValidateGpuIndex(uint32_t gpu_index); @@ -76,6 +78,6 @@ public: ValidateDbURI(const std::string &uri); }; -} -} -} \ No newline at end of file +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/DataTransfer.cpp b/cpp/src/wrapper/DataTransfer.cpp index 294a3c20ab..c6603b1991 100644 --- a/cpp/src/wrapper/DataTransfer.cpp +++ b/cpp/src/wrapper/DataTransfer.cpp @@ -16,45 +16,46 @@ // under the License. -#include "DataTransfer.h" +#include "wrapper/DataTransfer.h" +#include +#include +#include namespace zilliz { namespace milvus { namespace engine { -using namespace zilliz::knowhere; - -DatasetPtr -GenDatasetWithIds(const int64_t &nb, const int64_t &dim, const float *xb, const long *ids) { +knowhere::DatasetPtr +GenDatasetWithIds(const int64_t &nb, const int64_t &dim, const float *xb, const int64_t *ids) { std::vector shape{nb, dim}; - auto tensor = ConstructFloatTensor((uint8_t *) xb, nb * dim * sizeof(float), shape); - std::vector tensors{tensor}; - std::vector tensor_fields{ConstructFloatField("data")}; - auto tensor_schema = std::make_shared(tensor_fields); + auto tensor = knowhere::ConstructFloatTensor((uint8_t *) xb, nb * dim * sizeof(float), shape); + std::vector tensors{tensor}; + std::vector tensor_fields{knowhere::ConstructFloatField("data")}; + auto tensor_schema = std::make_shared(tensor_fields); - auto id_array = ConstructInt64Array((uint8_t *) ids, nb * sizeof(int64_t)); - std::vector arrays{id_array}; - std::vector array_fields{ConstructInt64Field("id")}; - auto array_schema = std::make_shared(tensor_fields); + auto id_array = knowhere::ConstructInt64Array((uint8_t *) ids, nb * sizeof(int64_t)); + std::vector arrays{id_array}; + std::vector array_fields{knowhere::ConstructInt64Field("id")}; + auto array_schema = std::make_shared(tensor_fields); - auto dataset = std::make_shared(std::move(arrays), array_schema, + auto dataset = std::make_shared(std::move(arrays), array_schema, std::move(tensors), tensor_schema); return dataset; } -DatasetPtr +knowhere::DatasetPtr GenDataset(const int64_t &nb, const int64_t &dim, const float *xb) { std::vector shape{nb, dim}; - auto tensor = ConstructFloatTensor((uint8_t *) xb, nb * dim * sizeof(float), shape); - std::vector tensors{tensor}; - std::vector tensor_fields{ConstructFloatField("data")}; - auto tensor_schema = std::make_shared(tensor_fields); + auto tensor = knowhere::ConstructFloatTensor((uint8_t *) xb, nb * dim * sizeof(float), shape); + std::vector tensors{tensor}; + std::vector tensor_fields{knowhere::ConstructFloatField("data")}; + auto tensor_schema = std::make_shared(tensor_fields); - auto dataset = std::make_shared(std::move(tensors), tensor_schema); + auto dataset = std::make_shared(std::move(tensors), tensor_schema); return dataset; } -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/DataTransfer.h b/cpp/src/wrapper/DataTransfer.h index 10aaf355c9..070adca055 100644 --- a/cpp/src/wrapper/DataTransfer.h +++ b/cpp/src/wrapper/DataTransfer.h @@ -20,17 +20,16 @@ #include "knowhere/adapter/Structure.h" - namespace zilliz { namespace milvus { namespace engine { extern zilliz::knowhere::DatasetPtr -GenDatasetWithIds(const int64_t &nb, const int64_t &dim, const float *xb, const long *ids); +GenDatasetWithIds(const int64_t &nb, const int64_t &dim, const float *xb, const int64_t *ids); extern zilliz::knowhere::DatasetPtr GenDataset(const int64_t &nb, const int64_t &dim, const float *xb); -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index 8ba38a77b0..b46acb1a4a 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -16,11 +16,15 @@ // under the License. -#include "KnowhereResource.h" +#include "wrapper/KnowhereResource.h" #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "server/Config.h" #include +#include +#include +#include +#include namespace zilliz { namespace milvus { @@ -31,20 +35,20 @@ constexpr int64_t M_BYTE = 1024 * 1024; Status KnowhereResource::Initialize() { struct GpuResourceSetting { - int64_t pinned_memory = 300*M_BYTE; - int64_t temp_memory = 300*M_BYTE; + int64_t pinned_memory = 300 * M_BYTE; + int64_t temp_memory = 300 * M_BYTE; int64_t resource_num = 2; }; - using GpuResourcesArray = std::map; + using GpuResourcesArray = std::map; GpuResourcesArray gpu_resources; Status s; //get build index gpu resource - server::Config& config = server::Config::GetInstance(); + server::Config &config = server::Config::GetInstance(); int32_t build_index_gpu; s = config.GetDBConfigBuildIndexGPU(build_index_gpu); - if (!s.ok()) return s; + if (!s.ok()) return s; gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); @@ -64,7 +68,7 @@ KnowhereResource::Initialize() { } //init gpu resources - for(auto iter = gpu_resources.begin(); iter != gpu_resources.end(); ++iter) { + for (auto iter = gpu_resources.begin(); iter != gpu_resources.end(); ++iter) { knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(iter->first, iter->second.pinned_memory, iter->second.temp_memory, @@ -80,6 +84,6 @@ KnowhereResource::Finalize() { return Status::OK(); } -} -} -} \ No newline at end of file +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/KnowhereResource.h b/cpp/src/wrapper/KnowhereResource.h index 9e8c27b5af..a8726f9542 100644 --- a/cpp/src/wrapper/KnowhereResource.h +++ b/cpp/src/wrapper/KnowhereResource.h @@ -25,7 +25,7 @@ namespace milvus { namespace engine { class KnowhereResource { -public: + public: static Status Initialize(); @@ -33,7 +33,6 @@ public: Finalize(); }; - -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/VecImpl.cpp b/cpp/src/wrapper/VecImpl.cpp index 24f6a2bb0e..bbc064aee5 100644 --- a/cpp/src/wrapper/VecImpl.cpp +++ b/cpp/src/wrapper/VecImpl.cpp @@ -16,27 +16,24 @@ // under the License. +#include "wrapper/VecImpl.h" #include "utils/Log.h" #include "knowhere/index/vector_index/IndexIDMAP.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/helpers/Cloner.h" -#include "VecImpl.h" #include "DataTransfer.h" - namespace zilliz { namespace milvus { namespace engine { -using namespace zilliz::knowhere; - Status -VecIndexImpl::BuildAll(const long &nb, +VecIndexImpl::BuildAll(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg, - const long &nt, + const int64_t &nt, const float *xt) { try { dim = cfg["dim"].as(); @@ -47,7 +44,7 @@ VecIndexImpl::BuildAll(const long &nb, auto model = index_->Train(dataset, cfg); index_->set_index_model(model); index_->Add(dataset, cfg); - } catch (KnowhereException &e) { + } catch (knowhere::KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (jsoncons::json_exception &e) { @@ -61,12 +58,12 @@ VecIndexImpl::BuildAll(const long &nb, } Status -VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config &cfg) { +VecIndexImpl::Add(const int64_t &nb, const float *xb, const int64_t *ids, const Config &cfg) { try { auto dataset = GenDatasetWithIds(nb, dim, xb, ids); index_->Add(dataset, cfg); - } catch (KnowhereException &e) { + } catch (knowhere::KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (jsoncons::json_exception &e) { @@ -80,7 +77,7 @@ VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config } Status -VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) { +VecIndexImpl::Search(const int64_t &nq, const float *xq, float *dist, int64_t *ids, const Config &cfg) { try { auto k = cfg["k"].as(); auto dataset = GenDataset(nq, dim, xq); @@ -116,8 +113,7 @@ VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, co // TODO(linxj): avoid copy here. memcpy(ids, p_ids, sizeof(int64_t) * nq * k); memcpy(dist, p_dist, sizeof(float) * nq * k); - - } catch (KnowhereException &e) { + } catch (knowhere::KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (jsoncons::json_exception &e) { @@ -186,7 +182,7 @@ VecIndexImpl::Clone() { int64_t VecIndexImpl::GetDeviceId() { - if (auto device_idx = std::dynamic_pointer_cast(index_)) { + if (auto device_idx = std::dynamic_pointer_cast(index_)) { return device_idx->GetGpuDevice(); } // else @@ -195,22 +191,22 @@ VecIndexImpl::GetDeviceId() { float * BFIndex::GetRawVectors() { - auto raw_index = std::dynamic_pointer_cast(index_); + auto raw_index = std::dynamic_pointer_cast(index_); if (raw_index) { return raw_index->GetRawVectors(); } return nullptr; } int64_t * BFIndex::GetRawIds() { - return std::static_pointer_cast(index_)->GetRawIds(); + return std::static_pointer_cast(index_)->GetRawIds(); } ErrorCode BFIndex::Build(const Config &cfg) { try { dim = cfg["dim"].as(); - std::static_pointer_cast(index_)->Train(cfg); - } catch (KnowhereException &e) { + std::static_pointer_cast(index_)->Train(cfg); + } catch (knowhere::KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return KNOWHERE_UNEXPECTED_ERROR; } catch (jsoncons::json_exception &e) { @@ -224,19 +220,19 @@ BFIndex::Build(const Config &cfg) { } Status -BFIndex::BuildAll(const long &nb, +BFIndex::BuildAll(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg, - const long &nt, + const int64_t &nt, const float *xt) { try { dim = cfg["dim"].as(); auto dataset = GenDatasetWithIds(nb, dim, xb, ids); - std::static_pointer_cast(index_)->Train(cfg); + std::static_pointer_cast(index_)->Train(cfg); index_->Add(dataset, cfg); - } catch (KnowhereException &e) { + } catch (knowhere::KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (jsoncons::json_exception &e) { @@ -251,11 +247,11 @@ BFIndex::BuildAll(const long &nb, // TODO(linxj): add lock here. Status -IVFMixIndex::BuildAll(const long &nb, +IVFMixIndex::BuildAll(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg, - const long &nt, + const int64_t &nt, const float *xt) { try { dim = cfg["dim"].as(); @@ -267,7 +263,7 @@ IVFMixIndex::BuildAll(const long &nb, index_->set_index_model(model); index_->Add(dataset, cfg); - if (auto device_index = std::dynamic_pointer_cast(index_)) { + if (auto device_index = std::dynamic_pointer_cast(index_)) { auto host_index = device_index->CopyGpuToCpu(Config()); index_ = host_index; type = ConvertToCpuIndexType(type); @@ -275,7 +271,7 @@ IVFMixIndex::BuildAll(const long &nb, WRAPPER_LOG_ERROR << "Build IVFMIXIndex Failed"; return Status(KNOWHERE_ERROR, "Build IVFMIXIndex Failed"); } - } catch (KnowhereException &e) { + } catch (knowhere::KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (jsoncons::json_exception &e) { @@ -296,6 +292,6 @@ IVFMixIndex::Load(const zilliz::knowhere::BinarySet &index_binary) { return Status::OK(); } -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/VecImpl.h b/cpp/src/wrapper/VecImpl.h index c111eb75b4..a91b24562b 100644 --- a/cpp/src/wrapper/VecImpl.h +++ b/cpp/src/wrapper/VecImpl.h @@ -21,6 +21,8 @@ #include "knowhere/index/vector_index/VectorIndex.h" #include "VecIndex.h" +#include +#include namespace zilliz { namespace milvus { @@ -29,14 +31,15 @@ namespace engine { class VecIndexImpl : public VecIndex { public: explicit VecIndexImpl(std::shared_ptr index, const IndexType &type) - : index_(std::move(index)), type(type) {}; + : index_(std::move(index)), type(type) { + } Status - BuildAll(const long &nb, + BuildAll(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg, - const long &nt, + const int64_t &nt, const float *xt) override; VecIndexPtr @@ -55,7 +58,7 @@ class VecIndexImpl : public VecIndex { Count() override; Status - Add(const long &nb, const float *xb, const long *ids, const Config &cfg) override; + Add(const int64_t &nb, const float *xb, const int64_t *ids, const Config &cfg) override; zilliz::knowhere::BinarySet Serialize() override; @@ -70,7 +73,7 @@ class VecIndexImpl : public VecIndex { GetDeviceId() override; Status - Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) override; + Search(const int64_t &nq, const float *xq, float *dist, int64_t *ids, const Config &cfg) override; protected: int64_t dim = 0; @@ -83,14 +86,15 @@ class VecIndexImpl : public VecIndex { class IVFMixIndex : public VecIndexImpl { public: explicit IVFMixIndex(std::shared_ptr index, const IndexType &type) - : VecIndexImpl(std::move(index), type) {}; + : VecIndexImpl(std::move(index), type) { + } Status - BuildAll(const long &nb, + BuildAll(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg, - const long &nt, + const int64_t &nt, const float *xt) override; Status @@ -100,7 +104,8 @@ class IVFMixIndex : public VecIndexImpl { class BFIndex : public VecIndexImpl { public: explicit BFIndex(std::shared_ptr index) : VecIndexImpl(std::move(index), - IndexType::FAISS_IDMAP) {}; + IndexType::FAISS_IDMAP) { + } ErrorCode Build(const Config &cfg); @@ -109,17 +114,17 @@ class BFIndex : public VecIndexImpl { GetRawVectors(); Status - BuildAll(const long &nb, + BuildAll(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg, - const long &nt, + const int64_t &nt, const float *xt) override; int64_t * GetRawIds(); }; -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/VecIndex.cpp b/cpp/src/wrapper/VecIndex.cpp index cd8843e5ac..c9fdbf8161 100644 --- a/cpp/src/wrapper/VecIndex.cpp +++ b/cpp/src/wrapper/VecIndex.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include "wrapper/VecIndex.h" #include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/index/vector_index/IndexIVFSQ.h" @@ -25,13 +26,11 @@ #include "knowhere/index/vector_index/IndexKDT.h" #include "knowhere/index/vector_index/IndexNSG.h" #include "knowhere/common/Exception.h" -#include "VecIndex.h" #include "VecImpl.h" #include "utils/Log.h" #include - namespace zilliz { namespace milvus { namespace engine { @@ -42,7 +41,7 @@ struct FileIOReader { std::fstream fs; std::string name; - FileIOReader(const std::string &fname); + explicit FileIOReader(const std::string &fname); ~FileIOReader(); @@ -72,13 +71,11 @@ FileIOReader::operator()(void *ptr, size_t size, size_t pos) { return 0; } - - struct FileIOWriter { std::fstream fs; std::string name; - FileIOWriter(const std::string &fname); + explicit FileIOWriter(const std::string &fname); ~FileIOWriter(); size_t operator()(void *ptr, size_t size); }; @@ -97,7 +94,6 @@ FileIOWriter::operator()(void *ptr, size_t size) { fs.write(reinterpret_cast(ptr), size); } - VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config &cfg) { std::shared_ptr index; @@ -215,7 +211,7 @@ write_index(VecIndexPtr index, const std::string &location) { FileIOWriter writer(location); writer(&index_type, sizeof(IndexType)); - for (auto &iter: binaryset.binary_map_) { + for (auto &iter : binaryset.binary_map_) { auto meta = iter.first.c_str(); size_t meta_length = iter.first.length(); writer(&meta_length, sizeof(meta_length)); @@ -242,15 +238,14 @@ write_index(VecIndexPtr index, const std::string &location) { return Status::OK(); } - // TODO(linxj): redo here. void -AutoGenParams(const IndexType &type, const long &size, zilliz::knowhere::Config &cfg) { +AutoGenParams(const IndexType &type, const int64_t &size, zilliz::knowhere::Config &cfg) { auto nlist = cfg.get_with_default("nlist", 0); if (size <= TYPICAL_COUNT / 16384 + 1) { //handle less row count, avoid nlist set to 0 cfg["nlist"] = 1; - } else if (int(size / TYPICAL_COUNT) * nlist == 0) { + } else if (int(size / TYPICAL_COUNT) *nlist == 0) { //calculate a proper nlist if nlist not specified or size less than TYPICAL_COUNT cfg["nlist"] = int(size / TYPICAL_COUNT * 16384); } @@ -341,7 +336,6 @@ ConvertToGpuIndexType(const IndexType &type) { } } - -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/VecIndex.h b/cpp/src/wrapper/VecIndex.h index 3737270f42..3e7eb0ffc4 100644 --- a/cpp/src/wrapper/VecIndex.h +++ b/cpp/src/wrapper/VecIndex.h @@ -25,7 +25,6 @@ #include "knowhere/common/Config.h" #include "knowhere/common/BinarySet.h" - namespace zilliz { namespace milvus { namespace engine { @@ -55,24 +54,24 @@ using VecIndexPtr = std::shared_ptr; class VecIndex { public: virtual Status - BuildAll(const long &nb, + BuildAll(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg, - const long &nt = 0, + const int64_t &nt = 0, const float *xt = nullptr) = 0; virtual Status - Add(const long &nb, + Add(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg = Config()) = 0; virtual Status - Search(const long &nq, + Search(const int64_t &nq, const float *xq, float *dist, - long *ids, + int64_t *ids, const Config &cfg = Config()) = 0; virtual VecIndexPtr @@ -117,7 +116,7 @@ extern VecIndexPtr LoadVecIndex(const IndexType &index_type, const zilliz::knowhere::BinarySet &index_binary); extern void -AutoGenParams(const IndexType &type, const long &size, Config &cfg); +AutoGenParams(const IndexType &type, const int64_t &size, Config &cfg); extern void ParameterValidation(const IndexType &type, Config &cfg); @@ -128,6 +127,6 @@ ConvertToCpuIndexType(const IndexType &type); extern IndexType ConvertToGpuIndexType(const IndexType &type); -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/unittest/metrics/prometheus_test.cpp b/cpp/unittest/metrics/prometheus_test.cpp index 6dbc05e948..a634a6ff9c 100644 --- a/cpp/unittest/metrics/prometheus_test.cpp +++ b/cpp/unittest/metrics/prometheus_test.cpp @@ -25,7 +25,7 @@ using namespace zilliz::milvus; TEST(PrometheusTest, PROMETHEUS_TEST){ - server::Config::GetInstance().SetMetricConfigAutoBootup("on"); + server::Config::GetInstance().SetMetricConfigEnableMonitor("on"); server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance(); instance.Init(); @@ -76,7 +76,7 @@ TEST(PrometheusTest, PROMETHEUS_TEST){ instance.GPUTemperature(); instance.CPUTemperature(); - server::Config::GetInstance().SetMetricConfigAutoBootup("off"); + server::Config::GetInstance().SetMetricConfigEnableMonitor("off"); instance.Init(); instance.CPUCoreUsagePercentSet(); instance.GPUTemperature(); diff --git a/cpp/unittest/server/appendix/log_config.conf b/cpp/unittest/server/appendix/log_config.conf index 29d46a7fe5..0a3e0d21af 100644 --- a/cpp/unittest/server/appendix/log_config.conf +++ b/cpp/unittest/server/appendix/log_config.conf @@ -1,19 +1,19 @@ * GLOBAL: FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-global.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-global.log" ENABLED = true TO_FILE = true TO_STANDARD_OUTPUT = false SUBSECOND_PRECISION = 3 PERFORMANCE_TRACKING = false - MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB + MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB * DEBUG: - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-debug.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-debug.log" ENABLED = true * WARNING: - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-warning.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-warning.log" * TRACE: - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-trace.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-trace.log" * VERBOSE: FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" TO_FILE = false @@ -21,7 +21,7 @@ ## Error logs * ERROR: ENABLED = true - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-error.log" * FATAL: ENABLED = true - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log" diff --git a/cpp/unittest/server/appendix/server_config.yaml b/cpp/unittest/server/appendix/server_config.yaml index e4b3de31b0..f92b2f1a18 100644 --- a/cpp/unittest/server/appendix/server_config.yaml +++ b/cpp/unittest/server/appendix/server_config.yaml @@ -1,45 +1,37 @@ +# All the following configurations are default values. + server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 - mode: single # milvus deployment type: single, cluster, read_only - time_zone: UTC+8 # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time + address: 0.0.0.0 # milvus server ip address (IPv4) + port: 19530 # port range: 1025 ~ 65534 + deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable + time_zone: UTC+8 db_config: - db_path: /tmp/milvus # milvus data storage path - db_slave_path: # secondry data storage path, split by semicolon + primary_path: /tmp/milvus # path used to store data and meta + secondary_path: # path used to store data only, split by semicolon - # URI format: dialect://username:password@host:port/database - # All parts except dialect are optional, but you MUST include the delimiters - # Currently dialect supports mysql or sqlite - db_backend_url: sqlite://:@:/ + backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database + # Keep 'dialect://:@:/', and replace other texts with real values. + # Replace 'dialect' with 'mysql' or 'sqlite' - archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB - archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day - insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB. - # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB - build_index_gpu: 0 # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1 + insert_buffer_size: 4 # GB, maximum insert buffer size allowed + build_index_gpu: 0 # gpu id used for building index metric_config: - is_startup: off # if monitoring start: on, off - collector: prometheus # metrics collector: prometheus - prometheus_config: # following are prometheus configure - port: 8080 # the port prometheus use to fetch metrics - push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address - push_gateway_port: 9091 # push method configure: push gateway port + enable_monitor: false # enable monitoring or not + collector: prometheus # prometheus + prometheus_config: + port: 8080 # port prometheus used to fetch metrics cache_config: - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory - cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 - insert_cache_immediately: false # insert data will be load into cache immediately for hot query - gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory - gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 + cpu_mem_capacity: 16 # GB, CPU memory used for cache + cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered + cache_insert_data: false # whether load inserted data into cache engine_config: - use_blas_threshold: 20 + blas_threshold: 20 resource_config: - mode: simple - resources: - # - cpu + resource_pool: + - cpu - gpu0 - diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp index f886e4f05a..b27d1e60c8 100644 --- a/cpp/unittest/server/config_test.cpp +++ b/cpp/unittest/server/config_test.cpp @@ -99,60 +99,18 @@ TEST(ConfigTest, CONFIG_TEST) { server_config.ClearSequences(); auto seqs = server_config.GetSequences(); ASSERT_TRUE(seqs.empty()); - - const server::ConfigNode const_node = root_config.GetChild("cache_config"); - float flt = const_node.GetFloatValue("cpu_cache_capacity"); - ASSERT_GT(flt, 0.0); } TEST(ConfigTest, SERVER_CONFIG_TEST) { - auto status = server::Config::GetInstance().LoadConfigFile(CONFIG_FILE_PATH); - ASSERT_TRUE(status.ok()); + server::Config& config = server::Config::GetInstance(); + Status s = config.LoadConfigFile(CONFIG_FILE_PATH); + ASSERT_TRUE(s.ok()); -// status = server::Config::GetInstance().ValidateConfig(); -// ASSERT_TRUE(status.ok()); -// -// const server::ServerConfig& config_const = config; -// server::ConfigNode node1 = config_const.GetConfig("server_config"); -// server::ConfigNode& node2 = config.GetConfig("cache_config"); -// node1.Combine(node2); -// -// int32_t cap = node1.GetInt32Value("cpu_cache_capacity"); -// ASSERT_GT(cap, 0); -// -// node1.SetValue("bool", "true"); -// bool bt = node1.GetBoolValue("bool"); -// ASSERT_TRUE(bt); + s = config.ValidateConfig(); + ASSERT_TRUE(s.ok()); -// server::Config::GetInstance().PrintAll(); -// -// unsigned long total_mem = 0, free_mem = 0; -// server::CommonUtil::GetSystemMemInfo(total_mem, free_mem); -// -// size_t gpu_mem = 0; -// server::ValidationUtil::GetGpuMemory(0, gpu_mem); -// -// server::ConfigNode& server_config = config.GetConfig("server_config"); -// server::ConfigNode& db_config = config.GetConfig("db_config"); -// server::ConfigNode& cache_config = config.GetConfig(server::CONFIG_CACHE); -// cache_config.SetValue(server::CACHE_FREE_PERCENT, "2.0"); -// status = config.ValidateConfig(); -// ASSERT_FALSE(status.ok()); -// -// size_t cache_cap = 16; -// size_t insert_buffer_size = (total_mem - cache_cap*GB + 1*GB)/GB; -// db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); -// cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap)); -// status = config.ValidateConfig(); -// ASSERT_FALSE(status.ok()); -// -// cache_cap = total_mem/GB + 2; -// cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap)); -// status = config.ValidateConfig(); -// ASSERT_FALSE(status.ok()); -// -// insert_buffer_size = total_mem/GB + 2; -// db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); -// status = config.ValidateConfig(); -// ASSERT_FALSE(status.ok()); + config.PrintAll(); + + s = config.ResetDefaultConfig(); + ASSERT_TRUE(s.ok()); } \ No newline at end of file diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index 6b27db2d48..7a88d6ff10 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -67,8 +67,8 @@ class RpcHandlerTest : public testing::Test { engine::DBOptions opt; server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/"); - server::Config::GetInstance().SetDBConfigPath("/tmp/milvus_test"); - server::Config::GetInstance().SetDBConfigSlavePath(""); + server::Config::GetInstance().SetDBConfigPrimaryPath("/tmp/milvus_test"); + server::Config::GetInstance().SetDBConfigSecondaryPath(""); server::Config::GetInstance().SetDBConfigArchiveDiskThreshold(""); server::Config::GetInstance().SetDBConfigArchiveDaysThreshold(""); server::Config::GetInstance().SetCacheConfigCacheInsertData("");