diff --git a/.clang-format b/.clang-format index 819c2d361e..1e124f7be8 100644 --- a/.clang-format +++ b/.clang-format @@ -18,3 +18,8 @@ BasedOnStyle: Google DerivePointerAlignment: false ColumnLimit: 120 +IndentWidth: 4 +AccessModifierOffset: -3 +AlwaysBreakAfterReturnType: All +AllowShortBlocksOnASingleLine: false +AllowShortFunctionsOnASingleLine: false diff --git a/cpp/build-support/code_style_clion.xml b/cpp/build-support/code_style_clion.xml new file mode 100644 index 0000000000..4c7dc4718c --- /dev/null +++ b/cpp/build-support/code_style_clion.xml @@ -0,0 +1,39 @@ + + + + + + \ No newline at end of file diff --git a/cpp/build.sh b/cpp/build.sh index 8b2f0b142b..69fd4f0f29 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -104,13 +104,13 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then fi echo "cpplint check passed!" -# # clang-format check -# make check-clang-format -# if [ $? -ne 0 ]; then -# echo "ERROR! clang-format check failed" -# exit 1 -# fi -# echo "clang-format check passed!" + # clang-format check + make check-clang-format + if [ $? -ne 0 ]; then + echo "ERROR! clang-format check failed" + exit 1 + fi + echo "clang-format check passed!" # # # clang-tidy check # make check-clang-tidy diff --git a/cpp/src/cache/Cache.h b/cpp/src/cache/Cache.h index b526e9339f..7818a2e6b7 100644 --- a/cpp/src/cache/Cache.h +++ b/cpp/src/cache/Cache.h @@ -15,55 +15,67 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "LRU.h" #include "utils/Log.h" -#include -#include #include +#include #include +#include namespace zilliz { namespace milvus { namespace cache { -template +template class Cache { public: - //mem_capacity, units:GB + // mem_capacity, units:GB Cache(int64_t capacity_gb, uint64_t cache_max_count); ~Cache() = default; - int64_t usage() const { + int64_t + usage() const { return usage_; } - int64_t capacity() const { + int64_t + capacity() const { return capacity_; - } //unit: BYTE - void set_capacity(int64_t capacity); //unit: BYTE + } // unit: BYTE + void + set_capacity(int64_t capacity); // unit: BYTE - double freemem_percent() const { + double + freemem_percent() const { return freemem_percent_; } - void set_freemem_percent(double percent) { + void + set_freemem_percent(double percent) { freemem_percent_ = percent; } - size_t size() const; - bool exists(const std::string &key); - ItemObj get(const std::string &key); - void insert(const std::string &key, const ItemObj &item); - void erase(const std::string &key); - void print(); - void clear(); + size_t + size() const; + bool + exists(const std::string& key); + ItemObj + get(const std::string& key); + void + insert(const std::string& key, const ItemObj& item); + void + erase(const std::string& key); + void + print(); + void + clear(); private: - void free_memory(); + void + free_memory(); private: int64_t usage_; @@ -74,8 +86,8 @@ class Cache { mutable std::mutex mutex_; }; -} // namespace cache -} // namespace milvus -} // namespace zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz #include "cache/Cache.inl" diff --git a/cpp/src/cache/CacheMgr.h b/cpp/src/cache/CacheMgr.h index 53004d10b2..11cd742a6a 100644 --- a/cpp/src/cache/CacheMgr.h +++ b/cpp/src/cache/CacheMgr.h @@ -15,40 +15,49 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "Cache.h" -#include "utils/Log.h" #include "metrics/Metrics.h" +#include "utils/Log.h" -#include #include +#include namespace zilliz { namespace milvus { namespace cache { -template +template class CacheMgr { public: - virtual uint64_t ItemCount() const; + virtual uint64_t + ItemCount() const; - virtual bool ItemExists(const std::string &key); + virtual bool + ItemExists(const std::string& key); - virtual ItemObj GetItem(const std::string &key); + virtual ItemObj + GetItem(const std::string& key); - virtual void InsertItem(const std::string &key, const ItemObj &data); + virtual void + InsertItem(const std::string& key, const ItemObj& data); - virtual void EraseItem(const std::string &key); + virtual void + EraseItem(const std::string& key); - virtual void PrintInfo(); + virtual void + PrintInfo(); - virtual void ClearCache(); + virtual void + ClearCache(); - int64_t CacheUsage() const; - int64_t CacheCapacity() const; - void SetCapacity(int64_t capacity); + int64_t + CacheUsage() const; + int64_t + CacheCapacity() const; + void + SetCapacity(int64_t capacity); protected: CacheMgr(); @@ -59,8 +68,8 @@ class CacheMgr { CachePtr cache_; }; -} // namespace cache -} // namespace milvus -} // namespace zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz #include "cache/CacheMgr.inl" diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index 7f53493017..99a284f074 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "cache/CpuCacheMgr.h" #include "server/Config.h" #include "utils/Log.h" @@ -31,7 +30,7 @@ constexpr int64_t unit = 1024 * 1024 * 1024; } CpuCacheMgr::CpuCacheMgr() { - server::Config &config = server::Config::GetInstance(); + server::Config& config = server::Config::GetInstance(); Status s; int32_t cpu_cache_cap; @@ -50,19 +49,19 @@ CpuCacheMgr::CpuCacheMgr() { if (cpu_cache_threshold > 0.0 && cpu_cache_threshold <= 1.0) { cache_->set_freemem_percent(cpu_cache_threshold); } else { - SERVER_LOG_ERROR << "Invalid cpu_cache_threshold: " << cpu_cache_threshold - << ", by default set to " << cache_->freemem_percent(); + SERVER_LOG_ERROR << "Invalid cpu_cache_threshold: " << cpu_cache_threshold << ", by default set to " + << cache_->freemem_percent(); } } -CpuCacheMgr * +CpuCacheMgr* CpuCacheMgr::GetInstance() { static CpuCacheMgr s_mgr; return &s_mgr; } engine::VecIndexPtr -CpuCacheMgr::GetIndex(const std::string &key) { +CpuCacheMgr::GetIndex(const std::string& key) { DataObjPtr obj = GetItem(key); if (obj != nullptr) { return obj->data(); @@ -71,6 +70,6 @@ CpuCacheMgr::GetIndex(const std::string &key) { return nullptr; } -} // namespace cache -} // namespace milvus -} // namespace zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/CpuCacheMgr.h b/cpp/src/cache/CpuCacheMgr.h index 32a83c4cc0..fbc4f6dfb3 100644 --- a/cpp/src/cache/CpuCacheMgr.h +++ b/cpp/src/cache/CpuCacheMgr.h @@ -20,8 +20,8 @@ #include "CacheMgr.h" #include "DataObj.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -32,12 +32,14 @@ class CpuCacheMgr : public CacheMgr { CpuCacheMgr(); public: - //TODO: use smart pointer instead - static CpuCacheMgr *GetInstance(); + // TODO: use smart pointer instead + static CpuCacheMgr* + GetInstance(); - engine::VecIndexPtr GetIndex(const std::string &key); + engine::VecIndexPtr + GetIndex(const std::string& key); }; -} // namespace cache -} // namespace milvus -} // namespace zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/DataObj.h b/cpp/src/cache/DataObj.h index 6ed3256eee..80cc560e7c 100644 --- a/cpp/src/cache/DataObj.h +++ b/cpp/src/cache/DataObj.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "src/wrapper/VecIndex.h" @@ -28,24 +27,24 @@ namespace cache { class DataObj { public: - explicit DataObj(const engine::VecIndexPtr &index) - : index_(index) { + explicit DataObj(const engine::VecIndexPtr& index) : index_(index) { } - DataObj(const engine::VecIndexPtr &index, int64_t size) - : index_(index), - size_(size) { + DataObj(const engine::VecIndexPtr& index, int64_t size) : index_(index), size_(size) { } - engine::VecIndexPtr data() { + engine::VecIndexPtr + data() { return index_; } - const engine::VecIndexPtr &data() const { + const engine::VecIndexPtr& + data() const { return index_; } - int64_t size() const { + int64_t + size() const { if (index_ == nullptr) { return 0; } @@ -64,6 +63,6 @@ class DataObj { using DataObjPtr = std::shared_ptr; -} // namespace cache -} // namespace milvus -} // namespace zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index f19e70de6b..9c9e94eda9 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. - #include "cache/GpuCacheMgr.h" -#include "utils/Log.h" #include "server/Config.h" +#include "utils/Log.h" #include #include @@ -35,7 +34,7 @@ constexpr int64_t G_BYTE = 1024 * 1024 * 1024; } GpuCacheMgr::GpuCacheMgr() { - server::Config &config = server::Config::GetInstance(); + server::Config& config = server::Config::GetInstance(); Status s; int32_t gpu_cache_cap; @@ -54,12 +53,12 @@ GpuCacheMgr::GpuCacheMgr() { if (gpu_mem_threshold > 0.0 && gpu_mem_threshold <= 1.0) { cache_->set_freemem_percent(gpu_mem_threshold); } else { - SERVER_LOG_ERROR << "Invalid gpu_mem_threshold: " << gpu_mem_threshold - << ", by default set to " << cache_->freemem_percent(); + SERVER_LOG_ERROR << "Invalid gpu_mem_threshold: " << gpu_mem_threshold << ", by default set to " + << cache_->freemem_percent(); } } -GpuCacheMgr * +GpuCacheMgr* GpuCacheMgr::GetInstance(uint64_t gpu_id) { if (instance_.find(gpu_id) == instance_.end()) { std::lock_guard lock(mutex_); @@ -74,7 +73,7 @@ GpuCacheMgr::GetInstance(uint64_t gpu_id) { } engine::VecIndexPtr -GpuCacheMgr::GetIndex(const std::string &key) { +GpuCacheMgr::GetIndex(const std::string& key) { DataObjPtr obj = GetItem(key); if (obj != nullptr) { return obj->data(); @@ -83,6 +82,6 @@ GpuCacheMgr::GetIndex(const std::string &key) { return nullptr; } -} // namespace cache -} // namespace milvus -} // namespace zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/GpuCacheMgr.h b/cpp/src/cache/GpuCacheMgr.h index 843a5ff67d..15cdcae28f 100644 --- a/cpp/src/cache/GpuCacheMgr.h +++ b/cpp/src/cache/GpuCacheMgr.h @@ -15,13 +15,12 @@ // specific language governing permissions and limitations // under the License. - #include "CacheMgr.h" #include "DataObj.h" -#include #include #include +#include namespace zilliz { namespace milvus { @@ -34,15 +33,17 @@ class GpuCacheMgr : public CacheMgr { public: GpuCacheMgr(); - static GpuCacheMgr *GetInstance(uint64_t gpu_id); + static GpuCacheMgr* + GetInstance(uint64_t gpu_id); - engine::VecIndexPtr GetIndex(const std::string &key); + engine::VecIndexPtr + GetIndex(const std::string& key); private: static std::mutex mutex_; static std::unordered_map instance_; }; -} // namespace cache -} // namespace milvus -} // namespace zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/LRU.h b/cpp/src/cache/LRU.h index 5446dd0f14..d6a5b9ace7 100644 --- a/cpp/src/cache/LRU.h +++ b/cpp/src/cache/LRU.h @@ -15,20 +15,19 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include -#include #include +#include #include +#include #include namespace zilliz { namespace milvus { namespace cache { -template +template class LRU { public: typedef typename std::pair key_value_pair_t; @@ -38,7 +37,8 @@ class LRU { explicit LRU(size_t max_size) : max_size_(max_size) { } - void put(const key_t &key, const value_t &value) { + void + put(const key_t& key, const value_t& value) { auto it = cache_items_map_.find(key); cache_items_list_.push_front(key_value_pair_t(key, value)); if (it != cache_items_map_.end()) { @@ -55,7 +55,8 @@ class LRU { } } - const value_t &get(const key_t &key) { + const value_t& + get(const key_t& key) { auto it = cache_items_map_.find(key); if (it == cache_items_map_.end()) { throw std::range_error("There is no such key in cache"); @@ -65,7 +66,8 @@ class LRU { } } - void erase(const key_t &key) { + void + erase(const key_t& key) { auto it = cache_items_map_.find(key); if (it != cache_items_map_.end()) { cache_items_list_.erase(it->second); @@ -73,32 +75,39 @@ class LRU { } } - bool exists(const key_t &key) const { + bool + exists(const key_t& key) const { return cache_items_map_.find(key) != cache_items_map_.end(); } - size_t size() const { + size_t + size() const { return cache_items_map_.size(); } - list_iterator_t begin() { + list_iterator_t + begin() { iter_ = cache_items_list_.begin(); return iter_; } - list_iterator_t end() { + list_iterator_t + end() { return cache_items_list_.end(); } - reverse_list_iterator_t rbegin() { + reverse_list_iterator_t + rbegin() { return cache_items_list_.rbegin(); } - reverse_list_iterator_t rend() { + reverse_list_iterator_t + rend() { return cache_items_list_.rend(); } - void clear() { + void + clear() { cache_items_list_.clear(); cache_items_map_.clear(); } @@ -110,7 +119,6 @@ class LRU { list_iterator_t iter_; }; -} // namespace cache -} // namespace milvus -} // namespace zilliz - +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/ConfigMgr.cpp b/cpp/src/config/ConfigMgr.cpp index a7cfdc9fda..e006217fd2 100644 --- a/cpp/src/config/ConfigMgr.cpp +++ b/cpp/src/config/ConfigMgr.cpp @@ -22,12 +22,12 @@ namespace zilliz { namespace milvus { namespace server { -ConfigMgr * +ConfigMgr* ConfigMgr::GetInstance() { static YamlConfigMgr mgr; return &mgr; } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/ConfigMgr.h b/cpp/src/config/ConfigMgr.h index 96a63aa2c8..e5336ee34d 100644 --- a/cpp/src/config/ConfigMgr.h +++ b/cpp/src/config/ConfigMgr.h @@ -17,8 +17,8 @@ #pragma once -#include "utils/Error.h" #include "ConfigNode.h" +#include "utils/Error.h" #include @@ -42,16 +42,22 @@ namespace server { class ConfigMgr { public: - static ConfigMgr *GetInstance(); + static ConfigMgr* + GetInstance(); - virtual ErrorCode LoadConfigFile(const std::string &filename) = 0; - virtual void Print() const = 0;//will be deleted - virtual std::string DumpString() const = 0; + virtual ErrorCode + LoadConfigFile(const std::string& filename) = 0; + virtual void + Print() const = 0; // will be deleted + virtual std::string + DumpString() const = 0; - virtual const ConfigNode &GetRootNode() const = 0; - virtual ConfigNode &GetRootNode() = 0; + virtual const ConfigNode& + GetRootNode() const = 0; + virtual ConfigNode& + GetRootNode() = 0; }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/ConfigNode.cpp b/cpp/src/config/ConfigNode.cpp index 87b080d572..ff27a0af75 100644 --- a/cpp/src/config/ConfigNode.cpp +++ b/cpp/src/config/ConfigNode.cpp @@ -19,51 +19,51 @@ #include "utils/Error.h" #include "utils/Log.h" +#include #include #include -#include namespace zilliz { namespace milvus { namespace server { void -ConfigNode::Combine(const ConfigNode &target) { - const std::map &kv = target.GetConfig(); +ConfigNode::Combine(const ConfigNode& target) { + const std::map& kv = target.GetConfig(); for (auto itr = kv.begin(); itr != kv.end(); ++itr) { config_[itr->first] = itr->second; } - const std::map > &sequences = target.GetSequences(); + const std::map >& sequences = target.GetSequences(); for (auto itr = sequences.begin(); itr != sequences.end(); ++itr) { sequences_[itr->first] = itr->second; } - const std::map &children = target.GetChildren(); + const std::map& children = target.GetChildren(); for (auto itr = children.begin(); itr != children.end(); ++itr) { children_[itr->first] = itr->second; } } -//key/value pair config +// key/value pair config void -ConfigNode::SetValue(const std::string &key, const std::string &value) { +ConfigNode::SetValue(const std::string& key, const std::string& value) { config_[key] = value; } std::string -ConfigNode::GetValue(const std::string ¶m_key, const std::string &default_val) const { +ConfigNode::GetValue(const std::string& param_key, const std::string& default_val) const { auto ref = config_.find(param_key); if (ref != config_.end()) { return ref->second; } - //THROW_UNEXPECTED_ERROR("Can't find parameter key: " + param_key); + // THROW_UNEXPECTED_ERROR("Can't find parameter key: " + param_key); return default_val; } bool -ConfigNode::GetBoolValue(const std::string ¶m_key, bool default_val) const { +ConfigNode::GetBoolValue(const std::string& param_key, bool default_val) const { std::string val = GetValue(param_key); if (!val.empty()) { std::transform(val.begin(), val.end(), val.begin(), ::tolower); @@ -74,17 +74,17 @@ ConfigNode::GetBoolValue(const std::string ¶m_key, bool default_val) const { } int32_t -ConfigNode::GetInt32Value(const std::string ¶m_key, int32_t default_val) const { +ConfigNode::GetInt32Value(const std::string& param_key, int32_t default_val) const { std::string val = GetValue(param_key); if (!val.empty()) { - return (int32_t) std::strtol(val.c_str(), nullptr, 10); + return (int32_t)std::strtol(val.c_str(), nullptr, 10); } else { return default_val; } } int64_t -ConfigNode::GetInt64Value(const std::string ¶m_key, int64_t default_val) const { +ConfigNode::GetInt64Value(const std::string& param_key, int64_t default_val) const { std::string val = GetValue(param_key); if (!val.empty()) { return std::strtol(val.c_str(), nullptr, 10); @@ -94,7 +94,7 @@ ConfigNode::GetInt64Value(const std::string ¶m_key, int64_t default_val) con } float -ConfigNode::GetFloatValue(const std::string ¶m_key, float default_val) const { +ConfigNode::GetFloatValue(const std::string& param_key, float default_val) const { std::string val = GetValue(param_key); if (!val.empty()) { return std::strtof(val.c_str(), nullptr); @@ -104,7 +104,7 @@ ConfigNode::GetFloatValue(const std::string ¶m_key, float default_val) const } double -ConfigNode::GetDoubleValue(const std::string ¶m_key, double default_val) const { +ConfigNode::GetDoubleValue(const std::string& param_key, double default_val) const { std::string val = GetValue(param_key); if (!val.empty()) { return std::strtod(val.c_str(), nullptr); @@ -113,7 +113,7 @@ ConfigNode::GetDoubleValue(const std::string ¶m_key, double default_val) con } } -const std::map & +const std::map& ConfigNode::GetConfig() const { return config_; } @@ -123,14 +123,14 @@ ConfigNode::ClearConfig() { config_.clear(); } -//key/object config +// key/object config void -ConfigNode::AddChild(const std::string &type_name, const ConfigNode &config) { +ConfigNode::AddChild(const std::string& type_name, const ConfigNode& config) { children_[type_name] = config; } ConfigNode -ConfigNode::GetChild(const std::string &type_name) const { +ConfigNode::GetChild(const std::string& type_name) const { auto ref = children_.find(type_name); if (ref != children_.end()) { return ref->second; @@ -140,20 +140,20 @@ ConfigNode::GetChild(const std::string &type_name) const { return nc; } -ConfigNode & -ConfigNode::GetChild(const std::string &type_name) { +ConfigNode& +ConfigNode::GetChild(const std::string& type_name) { return children_[type_name]; } void -ConfigNode::GetChildren(ConfigNodeArr &arr) const { +ConfigNode::GetChildren(ConfigNodeArr& arr) const { arr.clear(); for (auto ref : children_) { arr.push_back(ref.second); } } -const std::map & +const std::map& ConfigNode::GetChildren() const { return children_; } @@ -163,14 +163,14 @@ ConfigNode::ClearChildren() { children_.clear(); } -//key/sequence config +// key/sequence config void -ConfigNode::AddSequenceItem(const std::string &key, const std::string &item) { +ConfigNode::AddSequenceItem(const std::string& key, const std::string& item) { sequences_[key].push_back(item); } std::vector -ConfigNode::GetSequence(const std::string &key) const { +ConfigNode::GetSequence(const std::string& key) const { auto itr = sequences_.find(key); if (itr != sequences_.end()) { return itr->second; @@ -180,7 +180,7 @@ ConfigNode::GetSequence(const std::string &key) const { } } -const std::map > & +const std::map >& ConfigNode::GetSequences() const { return sequences_; } @@ -191,40 +191,40 @@ ConfigNode::ClearSequences() { } void -ConfigNode::PrintAll(const std::string &prefix) const { - for (auto &elem : config_) { +ConfigNode::PrintAll(const std::string& prefix) const { + for (auto& elem : config_) { SERVER_LOG_INFO << prefix << elem.first + ": " << elem.second; } - for (auto &elem : sequences_) { + for (auto& elem : sequences_) { SERVER_LOG_INFO << prefix << elem.first << ": "; - for (auto &str : elem.second) { + for (auto& str : elem.second) { SERVER_LOG_INFO << prefix << " - " << str; } } - for (auto &elem : children_) { + for (auto& elem : children_) { SERVER_LOG_INFO << prefix << elem.first << ": "; elem.second.PrintAll(prefix + " "); } } std::string -ConfigNode::DumpString(const std::string &prefix) const { +ConfigNode::DumpString(const std::string& prefix) const { std::stringstream str_buffer; const std::string endl = "\n"; - for (auto &elem : config_) { + for (auto& elem : config_) { str_buffer << prefix << elem.first << ": " << elem.second << endl; } - for (auto &elem : sequences_) { + for (auto& elem : sequences_) { str_buffer << prefix << elem.first << ": " << endl; - for (auto &str : elem.second) { + for (auto& str : elem.second) { str_buffer << prefix + " - " << str << endl; } } - for (auto &elem : children_) { + for (auto& elem : children_) { str_buffer << prefix << elem.first << ": " << endl; str_buffer << elem.second.DumpString(prefix + " ") << endl; } @@ -232,6 +232,6 @@ ConfigNode::DumpString(const std::string &prefix) const { return str_buffer.str(); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/ConfigNode.h b/cpp/src/config/ConfigNode.h index d3fabc0655..937066dbf5 100644 --- a/cpp/src/config/ConfigNode.h +++ b/cpp/src/config/ConfigNode.h @@ -17,9 +17,9 @@ #pragma once -#include -#include #include +#include +#include namespace zilliz { namespace milvus { @@ -30,39 +30,61 @@ typedef std::vector ConfigNodeArr; class ConfigNode { public: - void Combine(const ConfigNode &target); + void + Combine(const ConfigNode& target); - //key/value pair config - void SetValue(const std::string &key, const std::string &value); + // key/value pair config + void + SetValue(const std::string& key, const std::string& value); - std::string GetValue(const std::string ¶m_key, const std::string &default_val = "") const; - bool GetBoolValue(const std::string ¶m_key, bool default_val = false) const; - int32_t GetInt32Value(const std::string ¶m_key, int32_t default_val = 0) const; - int64_t GetInt64Value(const std::string ¶m_key, int64_t default_val = 0) const; - float GetFloatValue(const std::string ¶m_key, float default_val = 0.0) const; - double GetDoubleValue(const std::string ¶m_key, double default_val = 0.0) const; + std::string + GetValue(const std::string& param_key, const std::string& default_val = "") const; + bool + GetBoolValue(const std::string& param_key, bool default_val = false) const; + int32_t + GetInt32Value(const std::string& param_key, int32_t default_val = 0) const; + int64_t + GetInt64Value(const std::string& param_key, int64_t default_val = 0) const; + float + GetFloatValue(const std::string& param_key, float default_val = 0.0) const; + double + GetDoubleValue(const std::string& param_key, double default_val = 0.0) const; - const std::map &GetConfig() const; - void ClearConfig(); + const std::map& + GetConfig() const; + void + ClearConfig(); - //key/object config - void AddChild(const std::string &type_name, const ConfigNode &config); - ConfigNode GetChild(const std::string &type_name) const; - ConfigNode &GetChild(const std::string &type_name); - void GetChildren(ConfigNodeArr &arr) const; + // key/object config + void + AddChild(const std::string& type_name, const ConfigNode& config); + ConfigNode + GetChild(const std::string& type_name) const; + ConfigNode& + GetChild(const std::string& type_name); + void + GetChildren(ConfigNodeArr& arr) const; - const std::map &GetChildren() const; - void ClearChildren(); + const std::map& + GetChildren() const; + void + ClearChildren(); - //key/sequence config - void AddSequenceItem(const std::string &key, const std::string &item); - std::vector GetSequence(const std::string &key) const; + // key/sequence config + void + AddSequenceItem(const std::string& key, const std::string& item); + std::vector + GetSequence(const std::string& key) const; - const std::map > &GetSequences() const; - void ClearSequences(); + const std::map >& + GetSequences() const; + void + ClearSequences(); - void PrintAll(const std::string &prefix = "") const; - std::string DumpString(const std::string &prefix = "") const; + void + PrintAll(const std::string& prefix = "") const; + std::string + DumpString(const std::string& prefix = "") const; private: std::map config_; @@ -70,6 +92,6 @@ class ConfigNode { std::map > sequences_; }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/YamlConfigMgr.cpp b/cpp/src/config/YamlConfigMgr.cpp index 1dd8a3d943..02653d6fad 100644 --- a/cpp/src/config/YamlConfigMgr.cpp +++ b/cpp/src/config/YamlConfigMgr.cpp @@ -25,7 +25,7 @@ namespace milvus { namespace server { ErrorCode -YamlConfigMgr::LoadConfigFile(const std::string &filename) { +YamlConfigMgr::LoadConfigFile(const std::string& filename) { struct stat directoryStat; int statOK = stat(filename.c_str(), &directoryStat); if (statOK != 0) { @@ -36,8 +36,7 @@ YamlConfigMgr::LoadConfigFile(const std::string &filename) { try { node_ = YAML::LoadFile(filename); LoadConfigNode(node_, config_); - } - catch (YAML::Exception &e) { + } catch (YAML::Exception& e) { SERVER_LOG_ERROR << "Failed to load config file: " << std::string(e.what()); return SERVER_UNEXPECTED_ERROR; } @@ -56,20 +55,18 @@ YamlConfigMgr::DumpString() const { return config_.DumpString(""); } -const ConfigNode & +const ConfigNode& YamlConfigMgr::GetRootNode() const { return config_; } -ConfigNode & +ConfigNode& YamlConfigMgr::GetRootNode() { return config_; } bool -YamlConfigMgr::SetConfigValue(const YAML::Node &node, - const std::string &key, - ConfigNode &config) { +YamlConfigMgr::SetConfigValue(const YAML::Node& node, const std::string& key, ConfigNode& config) { if (node[key].IsDefined()) { config.SetValue(key, node[key].as()); return true; @@ -78,9 +75,7 @@ YamlConfigMgr::SetConfigValue(const YAML::Node &node, } bool -YamlConfigMgr::SetChildConfig(const YAML::Node &node, - const std::string &child_name, - ConfigNode &config) { +YamlConfigMgr::SetChildConfig(const YAML::Node& node, const std::string& child_name, ConfigNode& config) { if (node[child_name].IsDefined()) { ConfigNode sub_config; LoadConfigNode(node[child_name], sub_config); @@ -91,9 +86,7 @@ YamlConfigMgr::SetChildConfig(const YAML::Node &node, } bool -YamlConfigMgr::SetSequence(const YAML::Node &node, - const std::string &child_name, - ConfigNode &config) { +YamlConfigMgr::SetSequence(const YAML::Node& node, const std::string& child_name, ConfigNode& config) { if (node[child_name].IsDefined()) { size_t cnt = node[child_name].size(); for (size_t i = 0; i < cnt; i++) { @@ -105,7 +98,7 @@ YamlConfigMgr::SetSequence(const YAML::Node &node, } void -YamlConfigMgr::LoadConfigNode(const YAML::Node &node, ConfigNode &config) { +YamlConfigMgr::LoadConfigNode(const YAML::Node& node, ConfigNode& config) { std::string key; for (YAML::const_iterator it = node.begin(); it != node.end(); ++it) { if (!it->first.IsNull()) { @@ -121,6 +114,6 @@ YamlConfigMgr::LoadConfigNode(const YAML::Node &node, ConfigNode &config) { } } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/YamlConfigMgr.h b/cpp/src/config/YamlConfigMgr.h index 6b84943bf8..9697ce3cae 100644 --- a/cpp/src/config/YamlConfigMgr.h +++ b/cpp/src/config/YamlConfigMgr.h @@ -21,8 +21,8 @@ #include "ConfigNode.h" #include "utils/Error.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -30,34 +30,36 @@ namespace server { class YamlConfigMgr : public ConfigMgr { public: - virtual ErrorCode LoadConfigFile(const std::string &filename); - virtual void Print() const; - virtual std::string DumpString() const; + virtual ErrorCode + LoadConfigFile(const std::string& filename); + virtual void + Print() const; + virtual std::string + DumpString() const; - virtual const ConfigNode &GetRootNode() const; - virtual ConfigNode &GetRootNode(); + virtual const ConfigNode& + GetRootNode() const; + virtual ConfigNode& + GetRootNode(); private: - bool SetConfigValue(const YAML::Node &node, - const std::string &key, - ConfigNode &config); - - bool SetChildConfig(const YAML::Node &node, - const std::string &name, - ConfigNode &config); + bool + SetConfigValue(const YAML::Node& node, const std::string& key, ConfigNode& config); bool - SetSequence(const YAML::Node &node, - const std::string &child_name, - ConfigNode &config); + SetChildConfig(const YAML::Node& node, const std::string& name, ConfigNode& config); - void LoadConfigNode(const YAML::Node &node, ConfigNode &config); + bool + SetSequence(const YAML::Node& node, const std::string& child_name, ConfigNode& config); + + void + LoadConfigNode(const YAML::Node& node, ConfigNode& config); private: YAML::Node node_; ConfigNode config_; }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/core/CMakeLists.txt b/cpp/src/core/CMakeLists.txt index bef3f5e2f6..f1a13b4520 100644 --- a/cpp/src/core/CMakeLists.txt +++ b/cpp/src/core/CMakeLists.txt @@ -46,9 +46,11 @@ if(NOT CMAKE_BUILD_TYPE) endif(NOT CMAKE_BUILD_TYPE) if(CMAKE_BUILD_TYPE STREQUAL "Release") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3") else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") endif() MESSAGE(STATUS "CMAKE_CXX_FLAGS" ${CMAKE_CXX_FLAGS}) diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index c2b5122d65..0f3bb639c2 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -24,6 +24,7 @@ #include "NSG.h" #include "knowhere/common/Exception.h" +#include "knowhere/common/Log.h" #include "knowhere/common/Timer.h" #include "NSGHelper.h" @@ -83,8 +84,9 @@ void NsgIndex::Build_with_ids(size_t nb, const float *data, const long *ids, con for (int i = 0; i < ntotal; ++i) { total_degree += nsg[i].size(); } - std::cout << "graph physical size: " << total_degree * sizeof(node_t) / 1024 / 1024; - std::cout << "average degree: " << total_degree / ntotal; + + KNOWHERE_LOG_DEBUG << "Graph physical size: " << total_degree * sizeof(node_t) / 1024 / 1024 << "m"; + KNOWHERE_LOG_DEBUG << "Average degree: " << total_degree / ntotal; ///// is_trained = true; diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index cc6ea5bdcc..5ad1c9e00e 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -29,7 +29,7 @@ set(util_srcs ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/ArrowAdapter.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/Exception.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/Timer.cpp - utils.cpp + ${CORE_SOURCE_DIR}/test/utils.cpp ) # @@ -52,18 +52,10 @@ target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) # set(idmap_srcs - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVF.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp ) if(NOT TARGET test_idmap) - add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${util_srcs}) + add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${ivf_srcs} ${util_srcs}) endif() target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) @@ -86,5 +78,5 @@ install(TARGETS test_idmap DESTINATION unittest) install(TARGETS test_kdt DESTINATION unittest) #add_subdirectory(faiss_ori) -#add_subdirectory(test_nsg) +add_subdirectory(test_nsg) diff --git a/cpp/src/core/test/test_nsg/CMakeLists.txt b/cpp/src/core/test/test_nsg/CMakeLists.txt index 17b62cd40d..393cc188b0 100644 --- a/cpp/src/core/test/test_nsg/CMakeLists.txt +++ b/cpp/src/core/test/test_nsg/CMakeLists.txt @@ -1,6 +1,6 @@ ############################## -include_directories(/usr/local/include/gperftools) -link_directories(/usr/local/lib) +#include_directories(/usr/local/include/gperftools) +#link_directories(/usr/local/lib) add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO) @@ -13,29 +13,15 @@ else () endif () message(${OpenMP_CXX_FLAGS}) -include_directories(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg) - -aux_source_directory(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src) +include_directories(${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/nsg) +aux_source_directory(${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/nsg nsg_src) set(interface_src - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp - ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp - ../utils.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNSG.cpp ) if(NOT TARGET test_nsg) - add_executable(test_nsg - test_nsg.cpp - ${interface_src} - ${nsg_src} - ${util_srcs} - ) + add_executable(test_nsg test_nsg.cpp ${interface_src} ${nsg_src} ${util_srcs} ${ivf_srcs}) endif() target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs}) diff --git a/cpp/src/core/test/test_nsg/test_nsg.cpp b/cpp/src/core/test/test_nsg/test_nsg.cpp index b1b1edef46..8f0187e0a3 100644 --- a/cpp/src/core/test/test_nsg/test_nsg.cpp +++ b/cpp/src/core/test/test_nsg/test_nsg.cpp @@ -19,10 +19,11 @@ #include #include -#include "knowhere/common/exception.h" -#include "knowhere/index/vector_index/gpu_ivf.h" -#include "knowhere/index/vector_index/nsg_index.h" -#include "knowhere/index/vector_index/nsg/nsg_io.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/FaissBaseIndex.h" +#include "knowhere/index/vector_index/IndexNSG.h" +#include "knowhere/index/vector_index/nsg/NSGIO.h" +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "../utils.h" @@ -32,16 +33,31 @@ using ::testing::TestWithParam; using ::testing::Values; using ::testing::Combine; -constexpr int64_t DEVICE_ID = 0; +constexpr int64_t DEVICE_ID = 1; -class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple> { + class NSGInterfaceTest : public DataGen, public ::testing::Test { protected: void SetUp() override { //Init_with_default(); FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024*1024*200, 1024*1024*600, 2); - Generate(256, 10000, 1); + Generate(256, 1000000, 1); index_ = std::make_shared(); - std::tie(train_cfg, search_cfg) = GetParam(); + + auto tmp_conf = std::make_shared(); + tmp_conf->gpu_id = DEVICE_ID; + tmp_conf->knng = 100; + tmp_conf->nprobe = 32; + tmp_conf->nlist = 16384; + tmp_conf->search_length = 60; + tmp_conf->out_degree = 70; + tmp_conf->candidate_pool_size = 500; + tmp_conf->metric_type = METRICTYPE::L2; + train_conf = tmp_conf; + + auto tmp2_conf = std::make_shared(); + tmp2_conf->k = k; + tmp2_conf->search_length = 30; + search_conf = tmp2_conf; } void TearDown() override { @@ -50,18 +66,10 @@ class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple index_; - Config train_cfg; - Config search_cfg; + Config train_conf; + Config search_conf; }; -INSTANTIATE_TEST_CASE_P(NSGparameters, NSGInterfaceTest, - Values(std::make_tuple( - // search length > out_degree - Config::object{{"nlist", 128}, {"nprobe", 50}, {"knng", 100}, {"metric_type", "L2"}, - {"search_length", 60}, {"out_degree", 70}, {"candidate_pool_size", 500}}, - Config::object{{"k", 20}, {"search_length", 30}})) -); - void AssertAnns(const DatasetPtr &result, const int &nq, const int &k) { @@ -71,17 +79,17 @@ void AssertAnns(const DatasetPtr &result, } } -TEST_P(NSGInterfaceTest, basic_test) { +TEST_F(NSGInterfaceTest, basic_test) { assert(!xb.empty()); - auto model = index_->Train(base_dataset, train_cfg); - auto result = index_->Search(query_dataset, search_cfg); + auto model = index_->Train(base_dataset, train_conf); + auto result = index_->Search(query_dataset, search_conf); AssertAnns(result, nq, k); auto binaryset = index_->Serialize(); auto new_index = std::make_shared(); new_index->Load(binaryset); - auto new_result = new_index->Search(query_dataset, Config::object{{"k", k}}); + auto new_result = new_index->Search(query_dataset, search_conf); AssertAnns(result, nq, k); ASSERT_EQ(index_->Count(), nb); diff --git a/cpp/src/db/Constants.h b/cpp/src/db/Constants.h index 2beb3c3a97..0feffadaf4 100644 --- a/cpp/src/db/Constants.h +++ b/cpp/src/db/Constants.h @@ -36,6 +36,6 @@ static constexpr uint64_t ONE_KB = K; static constexpr uint64_t ONE_MB = ONE_KB * ONE_KB; static constexpr uint64_t ONE_GB = ONE_KB * ONE_MB; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 127812fc30..9a6a2e74a6 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "Options.h" @@ -23,8 +22,8 @@ #include "meta/Meta.h" #include "utils/Status.h" -#include #include +#include #include namespace zilliz { @@ -36,47 +35,65 @@ class Env; class DB { public: DB() = default; - DB(const DB &) = delete; - DB &operator=(const DB &) = delete; + DB(const DB&) = delete; + DB& + operator=(const DB&) = delete; virtual ~DB() = default; - virtual Status Start() = 0; - virtual Status Stop() = 0; + virtual Status + Start() = 0; + virtual Status + Stop() = 0; - virtual Status CreateTable(meta::TableSchema &table_schema_) = 0; - virtual Status DeleteTable(const std::string &table_id, const meta::DatesT &dates) = 0; - virtual Status DescribeTable(meta::TableSchema &table_schema_) = 0; - virtual Status HasTable(const std::string &table_id, bool &has_or_not_) = 0; - virtual Status AllTables(std::vector &table_schema_array) = 0; - virtual Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) = 0; - virtual Status PreloadTable(const std::string &table_id) = 0; - virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0; + virtual Status + CreateTable(meta::TableSchema& table_schema_) = 0; + virtual Status + DeleteTable(const std::string& table_id, const meta::DatesT& dates) = 0; + virtual Status + DescribeTable(meta::TableSchema& table_schema_) = 0; + virtual Status + HasTable(const std::string& table_id, bool& has_or_not_) = 0; + virtual Status + AllTables(std::vector& table_schema_array) = 0; + virtual Status + GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0; + virtual Status + PreloadTable(const std::string& table_id) = 0; + virtual Status + UpdateTableFlag(const std::string& table_id, int64_t flag) = 0; - virtual Status InsertVectors(const std::string &table_id_, - uint64_t n, const float *vectors, IDNumbers &vector_ids_) = 0; + virtual Status + InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0; - virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float *vectors, QueryResults &results) = 0; + virtual Status + Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, + QueryResults& results) = 0; - virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float *vectors, const meta::DatesT &dates, QueryResults &results) = 0; + virtual Status + Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, + const meta::DatesT& dates, QueryResults& results) = 0; - virtual Status Query(const std::string &table_id, const std::vector &file_ids, - uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors, - const meta::DatesT &dates, QueryResults &results) = 0; + virtual Status + Query(const std::string& table_id, const std::vector& file_ids, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0; - virtual Status Size(uint64_t &result) = 0; + virtual Status + Size(uint64_t& result) = 0; - virtual Status CreateIndex(const std::string &table_id, const TableIndex &index) = 0; - virtual Status DescribeIndex(const std::string &table_id, TableIndex &index) = 0; - virtual Status DropIndex(const std::string &table_id) = 0; + virtual Status + CreateIndex(const std::string& table_id, const TableIndex& index) = 0; + virtual Status + DescribeIndex(const std::string& table_id, TableIndex& index) = 0; + virtual Status + DropIndex(const std::string& table_id) = 0; - virtual Status DropAll() = 0; -}; // DB + virtual Status + DropAll() = 0; +}; // DB using DBPtr = std::shared_ptr; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/DBFactory.cpp b/cpp/src/db/DBFactory.cpp index edf70cccf8..89d04b3632 100644 --- a/cpp/src/db/DBFactory.cpp +++ b/cpp/src/db/DBFactory.cpp @@ -15,18 +15,17 @@ // specific language governing permissions and limitations // under the License. - #include "db/DBFactory.h" #include "DBImpl.h" -#include "utils/Exception.h" #include "meta/MetaFactory.h" -#include "meta/SqliteMetaImpl.h" #include "meta/MySQLMetaImpl.h" +#include "meta/SqliteMetaImpl.h" +#include "utils/Exception.h" #include #include -#include #include +#include #include namespace zilliz { @@ -42,10 +41,10 @@ DBFactory::BuildOption() { } DBPtr -DBFactory::Build(const DBOptions &options) { +DBFactory::Build(const DBOptions& options) { return std::make_shared(options); } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/DBFactory.h b/cpp/src/db/DBFactory.h index 1223a11c29..c5c73a0649 100644 --- a/cpp/src/db/DBFactory.h +++ b/cpp/src/db/DBFactory.h @@ -20,8 +20,8 @@ #include "DB.h" #include "Options.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -29,11 +29,13 @@ namespace engine { class DBFactory { public: - static DBOptions BuildOption(); + static DBOptions + BuildOption(); - static DBPtr Build(const DBOptions &options); + static DBPtr + Build(const DBOptions& options); }; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index ab3f3104af..2780d0f763 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -16,28 +16,28 @@ // under the License. #include "db/DBImpl.h" +#include "Utils.h" #include "cache/CpuCacheMgr.h" #include "cache/GpuCacheMgr.h" #include "engine/EngineFactory.h" #include "insert/MemMenagerFactory.h" -#include "meta/SqliteMetaImpl.h" -#include "meta/MetaFactory.h" #include "meta/MetaConsts.h" +#include "meta/MetaFactory.h" +#include "meta/SqliteMetaImpl.h" #include "metrics/Metrics.h" -#include "scheduler/job/SearchJob.h" -#include "scheduler/job/DeleteJob.h" #include "scheduler/SchedInst.h" -#include "utils/TimeRecorder.h" +#include "scheduler/job/DeleteJob.h" +#include "scheduler/job/SearchJob.h" #include "utils/Log.h" -#include "Utils.h" +#include "utils/TimeRecorder.h" #include -#include -#include -#include -#include #include #include +#include +#include +#include +#include namespace zilliz { namespace milvus { @@ -49,13 +49,10 @@ constexpr uint64_t METRIC_ACTION_INTERVAL = 1; constexpr uint64_t COMPACT_ACTION_INTERVAL = 1; constexpr uint64_t INDEX_ACTION_INTERVAL = 1; -} // namespace +} // namespace -DBImpl::DBImpl(const DBOptions &options) - : options_(options), - shutting_down_(true), - compact_thread_pool_(1, 1), - index_thread_pool_(1, 1) { +DBImpl::DBImpl(const DBOptions& options) + : options_(options), shutting_down_(true), compact_thread_pool_(1, 1), index_thread_pool_(1, 1) { meta_ptr_ = MetaFactory::Build(options.meta_, options.mode_); mem_mgr_ = MemManagerFactory::Build(meta_ptr_, options_); Start(); @@ -66,7 +63,7 @@ DBImpl::~DBImpl() { } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -//external api +// external api /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// Status DBImpl::Start() { @@ -77,7 +74,7 @@ DBImpl::Start() { ENGINE_LOG_TRACE << "DB service start"; shutting_down_.store(false, std::memory_order_release); - //for distribute version, some nodes are read only + // for distribute version, some nodes are read only if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { ENGINE_LOG_TRACE << "StartTimerTasks"; bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this); @@ -94,10 +91,10 @@ DBImpl::Stop() { shutting_down_.store(true, std::memory_order_release); - //makesure all memory data serialized + // makesure all memory data serialized MemSerialize(); - //wait compaction/buildindex finish + // wait compaction/buildindex finish bg_timer_thread_.join(); if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { @@ -114,30 +111,30 @@ DBImpl::DropAll() { } Status -DBImpl::CreateTable(meta::TableSchema &table_schema) { +DBImpl::CreateTable(meta::TableSchema& table_schema) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } meta::TableSchema temp_schema = table_schema; - temp_schema.index_file_size_ *= ONE_MB; //store as MB + temp_schema.index_file_size_ *= ONE_MB; // store as MB return meta_ptr_->CreateTable(temp_schema); } Status -DBImpl::DeleteTable(const std::string &table_id, const meta::DatesT &dates) { +DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } - //dates partly delete files of the table but currently we don't support + // dates partly delete files of the table but currently we don't support ENGINE_LOG_DEBUG << "Prepare to delete table " << table_id; if (dates.empty()) { - mem_mgr_->EraseMemVector(table_id); //not allow insert - meta_ptr_->DeleteTable(table_id); //soft delete table + mem_mgr_->EraseMemVector(table_id); // not allow insert + meta_ptr_->DeleteTable(table_id); // soft delete table - //scheduler will determine when to delete table files + // scheduler will determine when to delete table files auto nres = scheduler::ResMgrInst::GetInstance()->GetNumOfComputeResource(); scheduler::DeleteJobPtr job = std::make_shared(0, table_id, meta_ptr_, nres); scheduler::JobMgrInst::GetInstance()->Put(job); @@ -150,18 +147,18 @@ DBImpl::DeleteTable(const std::string &table_id, const meta::DatesT &dates) { } Status -DBImpl::DescribeTable(meta::TableSchema &table_schema) { +DBImpl::DescribeTable(meta::TableSchema& table_schema) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } auto stat = meta_ptr_->DescribeTable(table_schema); - table_schema.index_file_size_ /= ONE_MB; //return as MB + table_schema.index_file_size_ /= ONE_MB; // return as MB return stat; } Status -DBImpl::HasTable(const std::string &table_id, bool &has_or_not) { +DBImpl::HasTable(const std::string& table_id, bool& has_or_not) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -170,7 +167,7 @@ DBImpl::HasTable(const std::string &table_id, bool &has_or_not) { } Status -DBImpl::AllTables(std::vector &table_schema_array) { +DBImpl::AllTables(std::vector& table_schema_array) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -179,7 +176,7 @@ DBImpl::AllTables(std::vector &table_schema_array) { } Status -DBImpl::PreloadTable(const std::string &table_id) { +DBImpl::PreloadTable(const std::string& table_id) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -198,13 +195,11 @@ DBImpl::PreloadTable(const std::string &table_id) { int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); int64_t available_size = cache_total - cache_usage; - for (auto &day_files : files) { - for (auto &file : day_files.second) { - ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, - file.location_, - (EngineType) file.engine_type_, - (MetricType) file.metric_type_, - file.nlist_); + for (auto& day_files : files) { + for (auto& file : day_files.second) { + ExecutionEnginePtr engine = + EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, + (MetricType)file.metric_type_, file.nlist_); if (engine == nullptr) { ENGINE_LOG_ERROR << "Invalid engine type"; return Status(DB_ERROR, "Invalid engine type"); @@ -215,9 +210,9 @@ DBImpl::PreloadTable(const std::string &table_id) { break; } else { try { - //step 1: load index + // step 1: load index engine->Load(true); - } catch (std::exception &ex) { + } catch (std::exception& ex) { std::string msg = "Pre-load table encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; return Status(DB_ERROR, msg); @@ -229,7 +224,7 @@ DBImpl::PreloadTable(const std::string &table_id) { } Status -DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { +DBImpl::UpdateTableFlag(const std::string& table_id, int64_t flag) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -238,7 +233,7 @@ DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { } Status -DBImpl::GetTableRowCount(const std::string &table_id, uint64_t &row_count) { +DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -247,9 +242,8 @@ DBImpl::GetTableRowCount(const std::string &table_id, uint64_t &row_count) { } Status -DBImpl::InsertVectors(const std::string &table_id_, - uint64_t n, const float *vectors, IDNumbers &vector_ids_) { -// ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache"; +DBImpl::InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) { + // ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache"; if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -257,21 +251,21 @@ DBImpl::InsertVectors(const std::string &table_id_, Status status; zilliz::milvus::server::CollectInsertMetrics metrics(n, status); status = mem_mgr_->InsertVectors(table_id_, n, vectors, vector_ids_); -// std::chrono::microseconds time_span = -// std::chrono::duration_cast(end_time - start_time); -// double average_time = double(time_span.count()) / n; + // std::chrono::microseconds time_span = + // std::chrono::duration_cast(end_time - start_time); + // double average_time = double(time_span.count()) / n; -// ENGINE_LOG_DEBUG << "Insert vectors to cache finished"; + // ENGINE_LOG_DEBUG << "Insert vectors to cache finished"; return status; } Status -DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) { +DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { { std::unique_lock lock(build_index_mutex_); - //step 1: check index difference + // step 1: check index difference TableIndex old_index; auto status = DescribeIndex(table_id, old_index); if (!status.ok()) { @@ -279,9 +273,9 @@ DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) { return status; } - //step 2: update index info + // step 2: update index info TableIndex new_index = index; - new_index.metric_type_ = old_index.metric_type_;//dont change metric type, it was defined by CreateTable + new_index.metric_type_ = old_index.metric_type_; // dont change metric type, it was defined by CreateTable if (!utils::IsSameIndex(old_index, new_index)) { DropIndex(table_id); @@ -293,26 +287,23 @@ DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) { } } - //step 3: let merge file thread finish - //to avoid duplicate data bug + // step 3: let merge file thread finish + // to avoid duplicate data bug WaitMergeFileFinish(); - //step 4: wait and build index - //for IDMAP type, only wait all NEW file converted to RAW file - //for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files + // step 4: wait and build index + // for IDMAP type, only wait all NEW file converted to RAW file + // for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files std::vector file_types; - if (index.engine_type_ == (int) EngineType::FAISS_IDMAP) { + if (index.engine_type_ == (int)EngineType::FAISS_IDMAP) { file_types = { - (int) meta::TableFileSchema::NEW, - (int) meta::TableFileSchema::NEW_MERGE, + (int)meta::TableFileSchema::NEW, (int)meta::TableFileSchema::NEW_MERGE, }; } else { file_types = { - (int) meta::TableFileSchema::RAW, - (int) meta::TableFileSchema::NEW, - (int) meta::TableFileSchema::NEW_MERGE, - (int) meta::TableFileSchema::NEW_INDEX, - (int) meta::TableFileSchema::TO_INDEX, + (int)meta::TableFileSchema::RAW, (int)meta::TableFileSchema::NEW, + (int)meta::TableFileSchema::NEW_MERGE, (int)meta::TableFileSchema::NEW_INDEX, + (int)meta::TableFileSchema::TO_INDEX, }; } @@ -322,7 +313,7 @@ DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) { while (!file_ids.empty()) { ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times; - if (index.engine_type_ != (int) EngineType::FAISS_IDMAP) { + if (index.engine_type_ != (int)EngineType::FAISS_IDMAP) { status = meta_ptr_->UpdateTableFilesToIndex(table_id); } @@ -335,19 +326,19 @@ DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) { } Status -DBImpl::DescribeIndex(const std::string &table_id, TableIndex &index) { +DBImpl::DescribeIndex(const std::string& table_id, TableIndex& index) { return meta_ptr_->DescribeTableIndex(table_id, index); } Status -DBImpl::DropIndex(const std::string &table_id) { +DBImpl::DropIndex(const std::string& table_id) { ENGINE_LOG_DEBUG << "Drop index for table: " << table_id; return meta_ptr_->DropTableIndex(table_id); } Status -DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float *vectors, QueryResults &results) { +DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, + QueryResults& results) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -359,46 +350,47 @@ DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t npr } Status -DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float *vectors, const meta::DatesT &dates, QueryResults &results) { +DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, + const meta::DatesT& dates, QueryResults& results) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } ENGINE_LOG_DEBUG << "Query by dates for table: " << table_id; - //get all table files from table + // get all table files from table meta::DatePartionedTableFilesSchema files; std::vector ids; auto status = meta_ptr_->FilesToSearch(table_id, ids, dates, files); - if (!status.ok()) { return status; } + if (!status.ok()) { + return status; + } meta::TableFilesSchema file_id_array; - for (auto &day_files : files) { - for (auto &file : day_files.second) { + for (auto& day_files : files) { + for (auto& file : day_files.second) { file_id_array.push_back(file); } } - cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info before query + cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info before query status = QueryAsync(table_id, file_id_array, k, nq, nprobe, vectors, dates, results); - cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info after query + cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info after query return status; } Status -DBImpl::Query(const std::string &table_id, const std::vector &file_ids, - uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors, - const meta::DatesT &dates, QueryResults &results) { +DBImpl::Query(const std::string& table_id, const std::vector& file_ids, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } ENGINE_LOG_DEBUG << "Query by file ids for table: " << table_id; - //get specified files + // get specified files std::vector ids; - for (auto &id : file_ids) { + for (auto& id : file_ids) { meta::TableFileSchema table_file; table_file.table_id_ = table_id; std::string::size_type sz; @@ -412,8 +404,8 @@ DBImpl::Query(const std::string &table_id, const std::vector &file_ } meta::TableFilesSchema file_id_array; - for (auto &day_files : files_array) { - for (auto &file : day_files.second) { + for (auto& day_files : files_array) { + for (auto& file : day_files.second) { file_id_array.push_back(file); } } @@ -422,14 +414,14 @@ DBImpl::Query(const std::string &table_id, const std::vector &file_ return Status(DB_ERROR, "Invalid file id"); } - cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info before query + cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info before query status = QueryAsync(table_id, file_id_array, k, nq, nprobe, vectors, dates, results); - cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info after query + cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info after query return status; } Status -DBImpl::Size(uint64_t &result) { +DBImpl::Size(uint64_t& result) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -438,58 +430,57 @@ DBImpl::Size(uint64_t &result) { } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -//internal methods +// internal methods /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// Status -DBImpl::QueryAsync(const std::string &table_id, const meta::TableFilesSchema &files, - uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors, - const meta::DatesT &dates, QueryResults &results) { +DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) { server::CollectQueryMetrics metrics(nq); TimeRecorder rc(""); - //step 1: get files to search - ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: " - << dates.size(); + // step 1: get files to search + ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() + << " date range count: " << dates.size(); scheduler::SearchJobPtr job = std::make_shared(0, k, nq, nprobe, vectors); - for (auto &file : files) { + for (auto& file : files) { scheduler::TableFileSchemaPtr file_ptr = std::make_shared(file); job->AddIndexFile(file_ptr); } - //step 2: put search task to scheduler + // step 2: put search task to scheduler scheduler::JobMgrInst::GetInstance()->Put(job); job->WaitResult(); if (!job->GetStatus().ok()) { return job->GetStatus(); } - //step 3: print time cost information -// double load_cost = context->LoadCost(); -// double search_cost = context->SearchCost(); -// double reduce_cost = context->ReduceCost(); -// std::string load_info = TimeRecorder::GetTimeSpanStr(load_cost); -// std::string search_info = TimeRecorder::GetTimeSpanStr(search_cost); -// std::string reduce_info = TimeRecorder::GetTimeSpanStr(reduce_cost); -// if(search_cost > 0.0 || reduce_cost > 0.0) { -// double total_cost = load_cost + search_cost + reduce_cost; -// double load_percent = load_cost/total_cost; -// double search_percent = search_cost/total_cost; -// double reduce_percent = reduce_cost/total_cost; -// -// ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info -// << " percent: " << load_percent*100 << "%"; -// ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info -// << " percent: " << search_percent*100 << "%"; -// ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info -// << " percent: " << reduce_percent*100 << "%"; -// } else { -// ENGINE_LOG_DEBUG << "Engine load cost: " << load_info -// << " search cost: " << search_info -// << " reduce cost: " << reduce_info; -// } + // step 3: print time cost information + // double load_cost = context->LoadCost(); + // double search_cost = context->SearchCost(); + // double reduce_cost = context->ReduceCost(); + // std::string load_info = TimeRecorder::GetTimeSpanStr(load_cost); + // std::string search_info = TimeRecorder::GetTimeSpanStr(search_cost); + // std::string reduce_info = TimeRecorder::GetTimeSpanStr(reduce_cost); + // if(search_cost > 0.0 || reduce_cost > 0.0) { + // double total_cost = load_cost + search_cost + reduce_cost; + // double load_percent = load_cost/total_cost; + // double search_percent = search_cost/total_cost; + // double reduce_percent = reduce_cost/total_cost; + // + // ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info + // << " percent: " << load_percent*100 << "%"; + // ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info + // << " percent: " << search_percent*100 << "%"; + // ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info + // << " percent: " << reduce_percent*100 << "%"; + // } else { + // ENGINE_LOG_DEBUG << "Engine load cost: " << load_info + // << " search cost: " << search_info + // << " reduce cost: " << reduce_info; + // } - //step 4: construct results + // step 4: construct results results = job->GetResult(); rc.ElapseFromBegin("Engine query totally cost"); @@ -520,7 +511,7 @@ DBImpl::BackgroundTimerTask() { void DBImpl::WaitMergeFileFinish() { std::lock_guard lck(compact_result_mutex_); - for (auto &iter : compact_thread_results_) { + for (auto& iter : compact_thread_results_) { iter.wait(); } } @@ -528,7 +519,7 @@ DBImpl::WaitMergeFileFinish() { void DBImpl::WaitBuildIndexFinish() { std::lock_guard lck(index_result_mutex_); - for (auto &iter : index_thread_results_) { + for (auto& iter : index_thread_results_) { iter.wait(); } } @@ -569,7 +560,7 @@ DBImpl::MemSerialize() { std::lock_guard lck(mem_serialize_mutex_); std::set temp_table_ids; mem_mgr_->Serialize(temp_table_ids); - for (auto &id : temp_table_ids) { + for (auto& id : temp_table_ids) { compact_table_ids_.insert(id); } @@ -588,10 +579,10 @@ DBImpl::StartCompactionTask() { return; } - //serialize memory data + // serialize memory data MemSerialize(); - //compactiong has been finished? + // compactiong has been finished? { std::lock_guard lck(compact_result_mutex_); if (!compact_thread_results_.empty()) { @@ -602,7 +593,7 @@ DBImpl::StartCompactionTask() { } } - //add new compaction task + // add new compaction task { std::lock_guard lck(compact_result_mutex_); if (compact_thread_results_.empty()) { @@ -614,11 +605,10 @@ DBImpl::StartCompactionTask() { } Status -DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date, - const meta::TableFilesSchema &files) { +DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files) { ENGINE_LOG_DEBUG << "Merge files for table: " << table_id; - //step 1: create table file + // step 1: create table file meta::TableFileSchema table_file; table_file.table_id_ = table_id; table_file.date_ = date; @@ -630,15 +620,15 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date, return status; } - //step 2: merge files + // step 2: merge files ExecutionEnginePtr index = - EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType) table_file.engine_type_, - (MetricType) table_file.metric_type_, table_file.nlist_); + EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_, + (MetricType)table_file.metric_type_, table_file.nlist_); meta::TableFilesSchema updated; int64_t index_size = 0; - for (auto &file : files) { + for (auto& file : files) { server::CollectMergeFilesMetrics metrics; index->Merge(file.location_); @@ -651,11 +641,11 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date, if (index_size >= file_schema.index_file_size_) break; } - //step 3: serialize to disk + // step 3: serialize to disk try { index->Serialize(); - } catch (std::exception &ex) { - //typical error: out of disk space or permition denied + } catch (std::exception& ex) { + // typical error: out of disk space or permition denied std::string msg = "Serialize merged index encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; @@ -669,12 +659,12 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date, return Status(DB_ERROR, msg); } - //step 4: update table files state - //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size - //else set file type to RAW, no need to build index - if (table_file.engine_type_ != (int) EngineType::FAISS_IDMAP) { - table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ? - meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + // step 4: update table files state + // if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size + // else set file type to RAW, no need to build index + if (table_file.engine_type_ != (int)EngineType::FAISS_IDMAP) { + table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ? meta::TableFileSchema::TO_INDEX + : meta::TableFileSchema::RAW; } else { table_file.file_type_ = meta::TableFileSchema::RAW; } @@ -682,8 +672,7 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date, table_file.row_count_ = index->Count(); updated.push_back(table_file); status = meta_ptr_->UpdateTableFiles(updated); - ENGINE_LOG_DEBUG << "New merged file " << table_file.file_id_ << - " of size " << index->PhysicalSize() << " bytes"; + ENGINE_LOG_DEBUG << "New merged file " << table_file.file_id_ << " of size " << index->PhysicalSize() << " bytes"; if (options_.insert_cache_immediately_) { index->Cache(); @@ -693,7 +682,7 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date, } Status -DBImpl::BackgroundMergeFiles(const std::string &table_id) { +DBImpl::BackgroundMergeFiles(const std::string& table_id) { meta::DatePartionedTableFilesSchema raw_files; auto status = meta_ptr_->FilesToMerge(table_id, raw_files); if (!status.ok()) { @@ -702,7 +691,7 @@ DBImpl::BackgroundMergeFiles(const std::string &table_id) { } bool has_merge = false; - for (auto &kv : raw_files) { + for (auto& kv : raw_files) { auto files = kv.second; if (files.size() < options_.merge_trigger_number_) { ENGINE_LOG_DEBUG << "Files number not greater equal than merge trigger number, skip merge action"; @@ -725,7 +714,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { ENGINE_LOG_TRACE << " Background compaction thread start"; Status status; - for (auto &table_id : table_ids) { + for (auto& table_id : table_ids) { status = BackgroundMergeFiles(table_id); if (!status.ok()) { ENGINE_LOG_ERROR << "Merge files for table " << table_id << " failed: " << status.ToString(); @@ -739,7 +728,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); - int ttl = 5 * meta::M_SEC;//default: file will be deleted after 5 minutes + int ttl = 5 * meta::M_SEC; // default: file will be deleted after 5 minutes if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { ttl = meta::D_SEC; } @@ -756,7 +745,7 @@ DBImpl::StartBuildIndexTask(bool force) { return; } - //build index has been finished? + // build index has been finished? { std::lock_guard lck(index_result_mutex_); if (!index_thread_results_.empty()) { @@ -767,52 +756,50 @@ DBImpl::StartBuildIndexTask(bool force) { } } - //add new build index task + // add new build index task { std::lock_guard lck(index_result_mutex_); if (index_thread_results_.empty()) { - index_thread_results_.push_back( - index_thread_pool_.enqueue(&DBImpl::BackgroundBuildIndex, this)); + index_thread_results_.push_back(index_thread_pool_.enqueue(&DBImpl::BackgroundBuildIndex, this)); } } } Status -DBImpl::BuildIndex(const meta::TableFileSchema &file) { - ExecutionEnginePtr to_index = - EngineFactory::Build(file.dimension_, file.location_, (EngineType) file.engine_type_, - (MetricType) file.metric_type_, file.nlist_); +DBImpl::BuildIndex(const meta::TableFileSchema& file) { + ExecutionEnginePtr to_index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, + (MetricType)file.metric_type_, file.nlist_); if (to_index == nullptr) { ENGINE_LOG_ERROR << "Invalid engine type"; return Status(DB_ERROR, "Invalid engine type"); } try { - //step 1: load index + // step 1: load index Status status = to_index->Load(options_.insert_cache_immediately_); if (!status.ok()) { ENGINE_LOG_ERROR << "Failed to load index file: " << status.ToString(); return status; } - //step 2: create table file + // step 2: create table file meta::TableFileSchema table_file; table_file.table_id_ = file.table_id_; table_file.date_ = file.date_; table_file.file_type_ = - meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path + meta::TableFileSchema::NEW_INDEX; // for multi-db-path, distribute index file averagely to each path status = meta_ptr_->CreateTableFile(table_file); if (!status.ok()) { ENGINE_LOG_ERROR << "Failed to create table file: " << status.ToString(); return status; } - //step 3: build index + // step 3: build index std::shared_ptr index; try { server::CollectBuildIndexMetrics metrics; - index = to_index->BuildIndex(table_file.location_, (EngineType) table_file.engine_type_); + index = to_index->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_); if (index == nullptr) { table_file.file_type_ = meta::TableFileSchema::TO_DELETE; status = meta_ptr_->UpdateTableFile(table_file); @@ -821,8 +808,8 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) { return status; } - } catch (std::exception &ex) { - //typical error: out of gpu memory + } catch (std::exception& ex) { + // typical error: out of gpu memory std::string msg = "BuildIndex encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; @@ -836,7 +823,7 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) { return Status(DB_ERROR, msg); } - //step 4: if table has been deleted, dont save index file + // step 4: if table has been deleted, dont save index file bool has_table = false; meta_ptr_->HasTable(file.table_id_, has_table); if (!has_table) { @@ -844,11 +831,11 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) { return Status::OK(); } - //step 5: save index file + // step 5: save index file try { index->Serialize(); - } catch (std::exception &ex) { - //typical error: out of disk space or permition denied + } catch (std::exception& ex) { + // typical error: out of disk space or permition denied std::string msg = "Serialize index encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; @@ -862,7 +849,7 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) { return Status(DB_ERROR, msg); } - //step 6: update meta + // step 6: update meta table_file.file_type_ = meta::TableFileSchema::INDEX; table_file.file_size_ = index->PhysicalSize(); table_file.row_count_ = index->Count(); @@ -873,15 +860,15 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) { meta::TableFilesSchema update_files = {table_file, origin_file}; status = meta_ptr_->UpdateTableFiles(update_files); if (status.ok()) { - ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size " - << index->PhysicalSize() << " bytes" + ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size " << index->PhysicalSize() + << " bytes" << " from file " << origin_file.file_id_; if (options_.insert_cache_immediately_) { index->Cache(); } } else { - //failed to update meta, mark the new file as to_delete, don't delete old file + // failed to update meta, mark the new file as to_delete, don't delete old file origin_file.file_type_ = meta::TableFileSchema::TO_INDEX; status = meta_ptr_->UpdateTableFile(origin_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << origin_file.file_id_ << " to to_index"; @@ -890,7 +877,7 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) { status = meta_ptr_->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; } - } catch (std::exception &ex) { + } catch (std::exception& ex) { std::string msg = "Build index encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; return Status(DB_ERROR, msg); @@ -907,7 +894,7 @@ DBImpl::BackgroundBuildIndex() { meta::TableFilesSchema to_index_files; meta_ptr_->FilesToIndex(to_index_files); Status status; - for (auto &file : to_index_files) { + for (auto& file : to_index_files) { status = BuildIndex(file); if (!status.ok()) { ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString(); @@ -922,6 +909,6 @@ DBImpl::BackgroundBuildIndex() { ENGINE_LOG_TRACE << "Background build index thread exit"; } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 344e290445..366408d8b0 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -19,18 +19,18 @@ #include "DB.h" #include "Types.h" -#include "utils/ThreadPool.h" #include "src/db/insert/MemManager.h" +#include "utils/ThreadPool.h" -#include -#include -#include #include -#include +#include #include +#include +#include #include -#include #include +#include +#include namespace zilliz { namespace milvus { @@ -44,92 +44,101 @@ class Meta; class DBImpl : public DB { public: - explicit DBImpl(const DBOptions &options); + explicit DBImpl(const DBOptions& options); ~DBImpl(); - Status Start() override; - Status Stop() override; - Status DropAll() override; + Status + Start() override; + Status + Stop() override; + Status + DropAll() override; - Status CreateTable(meta::TableSchema &table_schema) override; + Status + CreateTable(meta::TableSchema& table_schema) override; - Status DeleteTable(const std::string &table_id, const meta::DatesT &dates) override; + Status + DeleteTable(const std::string& table_id, const meta::DatesT& dates) override; - Status DescribeTable(meta::TableSchema &table_schema) override; + Status + DescribeTable(meta::TableSchema& table_schema) override; - Status HasTable(const std::string &table_id, bool &has_or_not) override; + Status + HasTable(const std::string& table_id, bool& has_or_not) override; - Status AllTables(std::vector &table_schema_array) override; + Status + AllTables(std::vector& table_schema_array) override; - Status PreloadTable(const std::string &table_id) override; + Status + PreloadTable(const std::string& table_id) override; - Status UpdateTableFlag(const std::string &table_id, int64_t flag); + Status + UpdateTableFlag(const std::string& table_id, int64_t flag); - Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) override; + Status + GetTableRowCount(const std::string& table_id, uint64_t& row_count) override; - Status InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override; + Status + InsertVectors(const std::string& table_id, uint64_t n, const float* vectors, IDNumbers& vector_ids) override; - Status CreateIndex(const std::string &table_id, const TableIndex &index) override; + Status + CreateIndex(const std::string& table_id, const TableIndex& index) override; - Status DescribeIndex(const std::string &table_id, TableIndex &index) override; + Status + DescribeIndex(const std::string& table_id, TableIndex& index) override; - Status DropIndex(const std::string &table_id) override; + Status + DropIndex(const std::string& table_id) override; - Status Query(const std::string &table_id, - uint64_t k, - uint64_t nq, - uint64_t nprobe, - const float *vectors, - QueryResults &results) override; + Status + Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, + QueryResults& results) override; - Status Query(const std::string &table_id, - uint64_t k, - uint64_t nq, - uint64_t nprobe, - const float *vectors, - const meta::DatesT &dates, - QueryResults &results) override; + Status + Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, + const meta::DatesT& dates, QueryResults& results) override; - Status Query(const std::string &table_id, - const std::vector &file_ids, - uint64_t k, - uint64_t nq, - uint64_t nprobe, - const float *vectors, - const meta::DatesT &dates, - QueryResults &results) override; + Status + Query(const std::string& table_id, const std::vector& file_ids, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) override; - Status Size(uint64_t &result) override; + Status + Size(uint64_t& result) override; private: - Status QueryAsync(const std::string &table_id, - const meta::TableFilesSchema &files, - uint64_t k, - uint64_t nq, - uint64_t nprobe, - const float *vectors, - const meta::DatesT &dates, - QueryResults &results); + Status + QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results); - void BackgroundTimerTask(); - void WaitMergeFileFinish(); - void WaitBuildIndexFinish(); + void + BackgroundTimerTask(); + void + WaitMergeFileFinish(); + void + WaitBuildIndexFinish(); - void StartMetricTask(); + void + StartMetricTask(); - void StartCompactionTask(); - Status MergeFiles(const std::string &table_id, - const meta::DateT &date, - const meta::TableFilesSchema &files); - Status BackgroundMergeFiles(const std::string &table_id); - void BackgroundCompaction(std::set table_ids); + void + StartCompactionTask(); + Status + MergeFiles(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files); + Status + BackgroundMergeFiles(const std::string& table_id); + void + BackgroundCompaction(std::set table_ids); - void StartBuildIndexTask(bool force = false); - void BackgroundBuildIndex(); + void + StartBuildIndexTask(bool force = false); + void + BackgroundBuildIndex(); - Status BuildIndex(const meta::TableFileSchema &); + Status + BuildIndex(const meta::TableFileSchema&); - Status MemSerialize(); + Status + MemSerialize(); private: const DBOptions options_; @@ -152,9 +161,8 @@ class DBImpl : public DB { std::list> index_thread_results_; std::mutex build_index_mutex_; -}; // DBImpl +}; // DBImpl - -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/IDGenerator.cpp b/cpp/src/db/IDGenerator.cpp index 78c88b2c03..b80e1ce354 100644 --- a/cpp/src/db/IDGenerator.cpp +++ b/cpp/src/db/IDGenerator.cpp @@ -17,8 +17,8 @@ #include "db/IDGenerator.h" -#include #include +#include #include namespace zilliz { @@ -32,13 +32,12 @@ constexpr size_t SimpleIDGenerator::MAX_IDS_PER_MICRO; IDNumber SimpleIDGenerator::GetNextIDNumber() { auto now = std::chrono::system_clock::now(); - auto micros = std::chrono::duration_cast( - now.time_since_epoch()).count(); + auto micros = std::chrono::duration_cast(now.time_since_epoch()).count(); return micros * MAX_IDS_PER_MICRO; } void -SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) { +SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) { if (n > MAX_IDS_PER_MICRO) { NextIDNumbers(n - MAX_IDS_PER_MICRO, ids); NextIDNumbers(MAX_IDS_PER_MICRO, ids); @@ -49,8 +48,7 @@ SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) { } auto now = std::chrono::system_clock::now(); - auto micros = std::chrono::duration_cast( - now.time_since_epoch()).count(); + auto micros = std::chrono::duration_cast(now.time_since_epoch()).count(); micros *= MAX_IDS_PER_MICRO; for (int pos = 0; pos < n; ++pos) { @@ -59,11 +57,11 @@ SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) { } void -SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers &ids) { +SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers& ids) { ids.clear(); NextIDNumbers(n, ids); } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/IDGenerator.h b/cpp/src/db/IDGenerator.h index 0785870c4c..f41f745864 100644 --- a/cpp/src/db/IDGenerator.h +++ b/cpp/src/db/IDGenerator.h @@ -28,16 +28,14 @@ namespace engine { class IDGenerator { public: - virtual - IDNumber GetNextIDNumber() = 0; + virtual IDNumber + GetNextIDNumber() = 0; virtual void - GetNextIDNumbers(size_t n, IDNumbers &ids) = 0; - - virtual - ~IDGenerator() = 0; -}; // IDGenerator + GetNextIDNumbers(size_t n, IDNumbers& ids) = 0; + virtual ~IDGenerator() = 0; +}; // IDGenerator class SimpleIDGenerator : public IDGenerator { public: @@ -47,16 +45,15 @@ class SimpleIDGenerator : public IDGenerator { GetNextIDNumber() override; void - GetNextIDNumbers(size_t n, IDNumbers &ids) override; + GetNextIDNumbers(size_t n, IDNumbers& ids) override; private: void - NextIDNumbers(size_t n, IDNumbers &ids); + NextIDNumbers(size_t n, IDNumbers& ids); static constexpr size_t MAX_IDS_PER_MICRO = 1000; -}; // SimpleIDGenerator +}; // SimpleIDGenerator - -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index 4c82f3f736..edaf69c9e6 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -19,28 +19,28 @@ #include "utils/Exception.h" #include "utils/Log.h" -#include #include +#include #include namespace zilliz { namespace milvus { namespace engine { -ArchiveConf::ArchiveConf(const std::string &type, const std::string &criterias) { +ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) { ParseType(type); ParseCritirias(criterias); } void -ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT &criterial) { - for (auto &pair : criterial) { +ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT& criterial) { + for (auto& pair : criterial) { criterias_[pair.first] = pair.second; } } void -ArchiveConf::ParseCritirias(const std::string &criterias) { +ArchiveConf::ParseCritirias(const std::string& criterias) { std::stringstream ss(criterias); std::vector tokens; @@ -50,7 +50,7 @@ ArchiveConf::ParseCritirias(const std::string &criterias) { return; } - for (auto &token : tokens) { + for (auto& token : tokens) { if (token.empty()) { continue; } @@ -68,13 +68,11 @@ ArchiveConf::ParseCritirias(const std::string &criterias) { try { auto value = std::stoi(kv[1]); criterias_[kv[0]] = value; - } - catch (std::out_of_range &) { + } catch (std::out_of_range&) { std::string msg = "Out of range: '" + kv[1] + "'"; ENGINE_LOG_ERROR << msg; throw InvalidArgumentException(msg); - } - catch (...) { + } catch (...) { std::string msg = "Invalid argument: '" + kv[1] + "'"; ENGINE_LOG_ERROR << msg; throw InvalidArgumentException(msg); @@ -83,7 +81,7 @@ ArchiveConf::ParseCritirias(const std::string &criterias) { } void -ArchiveConf::ParseType(const std::string &type) { +ArchiveConf::ParseType(const std::string& type) { if (type != "delete" && type != "swap") { std::string msg = "Invalid argument: type='" + type + "'"; throw InvalidArgumentException(msg); @@ -91,6 +89,6 @@ ArchiveConf::ParseType(const std::string &type) { type_ = type; } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index d8be2767df..df3b1fc0c5 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -19,9 +19,9 @@ #include "Constants.h" -#include -#include #include +#include +#include #include namespace zilliz { @@ -30,27 +30,32 @@ namespace engine { class Env; -static const char *ARCHIVE_CONF_DISK = "disk"; -static const char *ARCHIVE_CONF_DAYS = "days"; +static const char* ARCHIVE_CONF_DISK = "disk"; +static const char* ARCHIVE_CONF_DAYS = "days"; struct ArchiveConf { using CriteriaT = std::map; - explicit ArchiveConf(const std::string &type, const std::string &criterias = std::string()); + explicit ArchiveConf(const std::string& type, const std::string& criterias = std::string()); - const std::string &GetType() const { + const std::string& + GetType() const { return type_; } - const CriteriaT GetCriterias() const { + const CriteriaT + GetCriterias() const { return criterias_; } - void SetCriterias(const ArchiveConf::CriteriaT &criterial); + void + SetCriterias(const ArchiveConf::CriteriaT& criterial); private: - void ParseCritirias(const std::string &type); - void ParseType(const std::string &criterias); + void + ParseCritirias(const std::string& type); + void + ParseType(const std::string& criterias); std::string type_; CriteriaT criterias_; @@ -61,14 +66,10 @@ struct DBMetaOptions { std::vector slave_paths_; std::string backend_uri_; ArchiveConf archive_conf_ = ArchiveConf("delete"); -}; // DBMetaOptions +}; // DBMetaOptions struct DBOptions { - typedef enum { - SINGLE = 0, - CLUSTER_READONLY, - CLUSTER_WRITABLE - } MODE; + typedef enum { SINGLE = 0, CLUSTER_READONLY, CLUSTER_WRITABLE } MODE; uint16_t merge_trigger_number_ = 2; DBMetaOptions meta_; @@ -76,9 +77,8 @@ struct DBOptions { size_t insert_buffer_size_ = 4 * ONE_GB; bool insert_cache_immediately_ = false; -}; // Options +}; // Options - -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/Types.h b/cpp/src/db/Types.h index 04bf680937..c37434a405 100644 --- a/cpp/src/db/Types.h +++ b/cpp/src/db/Types.h @@ -19,27 +19,27 @@ #include "db/engine/ExecutionEngine.h" -#include #include #include +#include namespace zilliz { namespace milvus { namespace engine { typedef int64_t IDNumber; -typedef IDNumber *IDNumberPtr; +typedef IDNumber* IDNumberPtr; typedef std::vector IDNumbers; typedef std::vector> QueryResult; typedef std::vector QueryResults; struct TableIndex { - int32_t engine_type_ = (int) EngineType::FAISS_IDMAP; + int32_t engine_type_ = (int)EngineType::FAISS_IDMAP; int32_t nlist_ = 16384; - int32_t metric_type_ = (int) MetricType::L2; + int32_t metric_type_ = (int)MetricType::L2; }; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index b3d72202cc..84e099b579 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -19,11 +19,11 @@ #include "utils/CommonUtil.h" #include "utils/Log.h" -#include +#include #include +#include #include #include -#include namespace zilliz { namespace milvus { @@ -32,20 +32,20 @@ namespace utils { namespace { -const char *TABLES_FOLDER = "/tables/"; +const char* TABLES_FOLDER = "/tables/"; uint64_t index_file_counter = 0; std::mutex index_file_counter_mutex; std::string -ConstructParentFolder(const std::string &db_path, const meta::TableFileSchema &table_file) { +ConstructParentFolder(const std::string& db_path, const meta::TableFileSchema& table_file) { std::string table_path = db_path + TABLES_FOLDER + table_file.table_id_; std::string partition_path = table_path + "/" + std::to_string(table_file.date_); return partition_path; } std::string -GetTableFileParentFolder(const DBMetaOptions &options, const meta::TableFileSchema &table_file) { +GetTableFileParentFolder(const DBMetaOptions& options, const meta::TableFileSchema& table_file) { uint64_t path_count = options.slave_paths_.size() + 1; std::string target_path = options.path_; uint64_t index = 0; @@ -70,19 +70,18 @@ GetTableFileParentFolder(const DBMetaOptions &options, const meta::TableFileSche return ConstructParentFolder(target_path, table_file); } -} // namespace +} // namespace int64_t GetMicroSecTimeStamp() { auto now = std::chrono::system_clock::now(); - auto micros = std::chrono::duration_cast( - now.time_since_epoch()).count(); + auto micros = std::chrono::duration_cast(now.time_since_epoch()).count(); return micros; } Status -CreateTablePath(const DBMetaOptions &options, const std::string &table_id) { +CreateTablePath(const DBMetaOptions& options, const std::string& table_id) { std::string db_path = options.path_; std::string table_path = db_path + TABLES_FOLDER + table_id; auto status = server::CommonUtil::CreateDirectory(table_path); @@ -91,7 +90,7 @@ CreateTablePath(const DBMetaOptions &options, const std::string &table_id) { return status; } - for (auto &path : options.slave_paths_) { + for (auto& path : options.slave_paths_) { table_path = path + TABLES_FOLDER + table_id; status = server::CommonUtil::CreateDirectory(table_path); if (!status.ok()) { @@ -104,17 +103,16 @@ CreateTablePath(const DBMetaOptions &options, const std::string &table_id) { } Status -DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force) { +DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force) { std::vector paths = options.slave_paths_; paths.push_back(options.path_); - for (auto &path : paths) { + for (auto& path : paths) { std::string table_path = path + TABLES_FOLDER + table_id; if (force) { boost::filesystem::remove_all(table_path); ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; - } else if (boost::filesystem::exists(table_path) && - boost::filesystem::is_empty(table_path)) { + } else if (boost::filesystem::exists(table_path) && boost::filesystem::is_empty(table_path)) { boost::filesystem::remove_all(table_path); ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; } @@ -124,7 +122,7 @@ DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool } Status -CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) { +CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { std::string parent_path = GetTableFileParentFolder(options, table_file); auto status = server::CommonUtil::CreateDirectory(parent_path); @@ -139,14 +137,14 @@ CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_f } Status -GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) { +GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { std::string parent_path = ConstructParentFolder(options.path_, table_file); std::string file_path = parent_path + "/" + table_file.file_id_; if (boost::filesystem::exists(file_path)) { table_file.location_ = file_path; return Status::OK(); } else { - for (auto &path : options.slave_paths_) { + for (auto& path : options.slave_paths_) { parent_path = ConstructParentFolder(path, table_file); file_path = parent_path + "/" + table_file.file_id_; if (boost::filesystem::exists(file_path)) { @@ -157,28 +155,26 @@ GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file } std::string msg = "Table file doesn't exist: " + file_path; - ENGINE_LOG_ERROR << msg << " in path: " << options.path_ - << " for table: " << table_file.table_id_; + ENGINE_LOG_ERROR << msg << " in path: " << options.path_ << " for table: " << table_file.table_id_; return Status(DB_ERROR, msg); } Status -DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) { +DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { utils::GetTableFilePath(options, table_file); boost::filesystem::remove(table_file.location_); return Status::OK(); } bool -IsSameIndex(const TableIndex &index1, const TableIndex &index2) { - return index1.engine_type_ == index2.engine_type_ - && index1.nlist_ == index2.nlist_ - && index1.metric_type_ == index2.metric_type_; +IsSameIndex(const TableIndex& index1, const TableIndex& index2) { + return index1.engine_type_ == index2.engine_type_ && index1.nlist_ == index2.nlist_ && + index1.metric_type_ == index2.metric_type_; } meta::DateT -GetDate(const std::time_t &t, int day_delta) { +GetDate(const std::time_t& t, int day_delta) { struct tm ltm; localtime_r(&t, <m); if (day_delta > 0) { @@ -211,20 +207,15 @@ GetDate() { // URI format: dialect://username:password@host:port/database Status -ParseMetaUri(const std::string &uri, MetaUriInfo &info) { +ParseMetaUri(const std::string& uri, MetaUriInfo& info) { std::string dialect_regex = "(.*)"; std::string username_tegex = "(.*)"; std::string password_regex = "(.*)"; std::string host_regex = "(.*)"; std::string port_regex = "(.*)"; std::string db_name_regex = "(.*)"; - std::string uri_regex_str = - dialect_regex + "\\:\\/\\/" + - username_tegex + "\\:" + - password_regex + "\\@" + - host_regex + "\\:" + - port_regex + "\\/" + - db_name_regex; + std::string uri_regex_str = dialect_regex + "\\:\\/\\/" + username_tegex + "\\:" + password_regex + "\\@" + + host_regex + "\\:" + port_regex + "\\/" + db_name_regex; std::regex uri_regex(uri_regex_str); std::smatch pieces_match; @@ -237,7 +228,7 @@ ParseMetaUri(const std::string &uri, MetaUriInfo &info) { info.port_ = pieces_match[5].str(); info.db_name_ = pieces_match[6].str(); - //TODO: verify host, port... + // TODO: verify host, port... } else { return Status(DB_INVALID_META_URI, "Invalid meta uri: " + uri); } @@ -245,7 +236,7 @@ ParseMetaUri(const std::string &uri, MetaUriInfo &info) { return Status::OK(); } -} // namespace utils -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace utils +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/Utils.h b/cpp/src/db/Utils.h index fbeca84686..c161e32b95 100644 --- a/cpp/src/db/Utils.h +++ b/cpp/src/db/Utils.h @@ -18,11 +18,11 @@ #pragma once #include "Options.h" -#include "db/meta/MetaTypes.h" #include "db/Types.h" +#include "db/meta/MetaTypes.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -33,22 +33,22 @@ int64_t GetMicroSecTimeStamp(); Status -CreateTablePath(const DBMetaOptions &options, const std::string &table_id); +CreateTablePath(const DBMetaOptions& options, const std::string& table_id); Status -DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force = true); +DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force = true); Status -CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file); +CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); Status -GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file); +GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); Status -DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file); +DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); bool -IsSameIndex(const TableIndex &index1, const TableIndex &index2); +IsSameIndex(const TableIndex& index1, const TableIndex& index2); meta::DateT -GetDate(const std::time_t &t, int day_delta = 0); +GetDate(const std::time_t& t, int day_delta = 0); meta::DateT GetDate(); meta::DateT @@ -64,9 +64,9 @@ struct MetaUriInfo { }; Status -ParseMetaUri(const std::string &uri, MetaUriInfo &info); +ParseMetaUri(const std::string& uri, MetaUriInfo& info); -} // namespace utils -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace utils +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/engine/EngineFactory.cpp b/cpp/src/db/engine/EngineFactory.cpp index 043744f803..338b7d72af 100644 --- a/cpp/src/db/engine/EngineFactory.cpp +++ b/cpp/src/db/engine/EngineFactory.cpp @@ -26,17 +26,14 @@ namespace milvus { namespace engine { ExecutionEnginePtr -EngineFactory::Build(uint16_t dimension, - const std::string &location, - EngineType index_type, - MetricType metric_type, +EngineFactory::Build(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type, int32_t nlist) { if (index_type == EngineType::INVALID) { ENGINE_LOG_ERROR << "Unsupported engine type"; return nullptr; } - ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int) index_type; + ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int)index_type; ExecutionEnginePtr execution_engine_ptr = std::make_shared(dimension, location, index_type, metric_type, nlist); @@ -44,6 +41,6 @@ EngineFactory::Build(uint16_t dimension, return execution_engine_ptr; } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/engine/EngineFactory.h b/cpp/src/db/engine/EngineFactory.h index 105e48f885..5d22fbb4d2 100644 --- a/cpp/src/db/engine/EngineFactory.h +++ b/cpp/src/db/engine/EngineFactory.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "ExecutionEngine.h" @@ -29,14 +28,11 @@ namespace engine { class EngineFactory { public: - static ExecutionEnginePtr Build(uint16_t dimension, - const std::string &location, - EngineType index_type, - MetricType metric_type, - int32_t nlist); + static ExecutionEnginePtr + Build(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type, + int32_t nlist); }; -} // namespace engine -} // namespace milvus -} // namespace zilliz - +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/engine/ExecutionEngine.h b/cpp/src/db/engine/ExecutionEngine.h index c63760686f..4ae3e872b3 100644 --- a/cpp/src/db/engine/ExecutionEngine.h +++ b/cpp/src/db/engine/ExecutionEngine.h @@ -19,9 +19,9 @@ #include "utils/Status.h" -#include #include #include +#include namespace zilliz { namespace milvus { @@ -43,52 +43,66 @@ enum class MetricType { class ExecutionEngine { public: - virtual Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) = 0; + virtual Status + AddWithIds(int64_t n, const float* xdata, const int64_t* xids) = 0; - virtual size_t Count() const = 0; + virtual size_t + Count() const = 0; - virtual size_t Size() const = 0; + virtual size_t + Size() const = 0; - virtual size_t Dimension() const = 0; + virtual size_t + Dimension() const = 0; - virtual size_t PhysicalSize() const = 0; + virtual size_t + PhysicalSize() const = 0; - virtual Status Serialize() = 0; + virtual Status + Serialize() = 0; - virtual Status Load(bool to_cache = true) = 0; + virtual Status + Load(bool to_cache = true) = 0; - virtual Status CopyToGpu(uint64_t device_id) = 0; + virtual Status + CopyToGpu(uint64_t device_id) = 0; - virtual Status CopyToCpu() = 0; + virtual Status + CopyToCpu() = 0; - virtual std::shared_ptr Clone() = 0; + virtual std::shared_ptr + Clone() = 0; - virtual Status Merge(const std::string &location) = 0; + virtual Status + Merge(const std::string& location) = 0; - virtual Status Search(int64_t n, - const float *data, - int64_t k, - int64_t nprobe, - float *distances, - int64_t *labels) const = 0; + virtual Status + Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances, int64_t* labels) const = 0; - virtual std::shared_ptr BuildIndex(const std::string &location, EngineType engine_type) = 0; + virtual std::shared_ptr + BuildIndex(const std::string& location, EngineType engine_type) = 0; - virtual Status Cache() = 0; + virtual Status + Cache() = 0; - virtual Status GpuCache(uint64_t gpu_id) = 0; + virtual Status + GpuCache(uint64_t gpu_id) = 0; - virtual Status Init() = 0; + virtual Status + Init() = 0; - virtual EngineType IndexEngineType() const = 0; + virtual EngineType + IndexEngineType() const = 0; - virtual MetricType IndexMetricType() const = 0; + virtual MetricType + IndexMetricType() const = 0; - virtual std::string GetLocation() const = 0; + virtual std::string + GetLocation() const = 0; }; using ExecutionEnginePtr = std::shared_ptr; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index a9937092fd..e8f9ba395d 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -16,20 +16,20 @@ // under the License. #include "db/engine/ExecutionEngineImpl.h" -#include "cache/GpuCacheMgr.h" #include "cache/CpuCacheMgr.h" +#include "cache/GpuCacheMgr.h" #include "metrics/Metrics.h" -#include "utils/Log.h" #include "utils/CommonUtil.h" #include "utils/Exception.h" +#include "utils/Log.h" -#include "src/wrapper/VecIndex.h" -#include "src/wrapper/VecImpl.h" -#include "knowhere/common/Exception.h" #include "knowhere/common/Config.h" -#include "wrapper/ConfAdapterMgr.h" -#include "wrapper/ConfAdapter.h" +#include "knowhere/common/Exception.h" #include "server/Config.h" +#include "src/wrapper/VecImpl.h" +#include "src/wrapper/VecIndex.h" +#include "wrapper/ConfAdapter.h" +#include "wrapper/ConfAdapterMgr.h" #include #include @@ -38,17 +38,9 @@ namespace zilliz { namespace milvus { namespace engine { -ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, - const std::string &location, - EngineType index_type, - MetricType metric_type, - int32_t nlist) - : location_(location), - dim_(dimension), - index_type_(index_type), - metric_type_(metric_type), - nlist_(nlist) { - +ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& location, EngineType index_type, + MetricType metric_type, int32_t nlist) + : location_(location), dim_(dimension), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) { index_ = CreatetVecIndex(EngineType::FAISS_IDMAP); if (!index_) { throw Exception(DB_ERROR, "Could not create VecIndex"); @@ -57,8 +49,7 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, TempMetaConf temp_conf; temp_conf.gpu_id = gpu_num_; temp_conf.dim = dimension; - temp_conf.metric_type = (metric_type_ == MetricType::IP) ? - knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2; + temp_conf.metric_type = (metric_type_ == MetricType::IP) ? knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2; auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType()); auto conf = adapter->Match(temp_conf); @@ -68,16 +59,9 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, } } -ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index, - const std::string &location, - EngineType index_type, - MetricType metric_type, - int32_t nlist) - : index_(std::move(index)), - location_(location), - index_type_(index_type), - metric_type_(metric_type), - nlist_(nlist) { +ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index, const std::string& location, EngineType index_type, + MetricType metric_type, int32_t nlist) + : index_(std::move(index)), location_(location), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) { } VecIndexPtr @@ -109,7 +93,7 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { } Status -ExecutionEngineImpl::AddWithIds(int64_t n, const float *xdata, const int64_t *xids) { +ExecutionEngineImpl::AddWithIds(int64_t n, const float* xdata, const int64_t* xids) { auto status = index_->Add(n, xdata, xids); return status; } @@ -125,7 +109,7 @@ ExecutionEngineImpl::Count() const { size_t ExecutionEngineImpl::Size() const { - return (size_t) (Count() * Dimension()) * sizeof(float); + return (size_t)(Count() * Dimension()) * sizeof(float); } size_t @@ -164,7 +148,7 @@ ExecutionEngineImpl::Load(bool to_cache) { } else { ENGINE_LOG_DEBUG << "Disk io from: " << location_; } - } catch (std::exception &e) { + } catch (std::exception& e) { ENGINE_LOG_ERROR << e.what(); return Status(DB_ERROR, e.what()); } @@ -191,7 +175,7 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id) { try { index_ = index_->CopyToGpu(device_id); ENGINE_LOG_DEBUG << "CPU to GPU" << device_id; - } catch (std::exception &e) { + } catch (std::exception& e) { ENGINE_LOG_ERROR << e.what(); return Status(DB_ERROR, e.what()); } @@ -219,7 +203,7 @@ ExecutionEngineImpl::CopyToCpu() { try { index_ = index_->CopyToCpu(); ENGINE_LOG_DEBUG << "GPU to CPU"; - } catch (std::exception &e) { + } catch (std::exception& e) { ENGINE_LOG_ERROR << e.what(); return Status(DB_ERROR, e.what()); } @@ -245,7 +229,7 @@ ExecutionEngineImpl::Clone() { } Status -ExecutionEngineImpl::Merge(const std::string &location) { +ExecutionEngineImpl::Merge(const std::string& location) { if (location == location_) { return Status(DB_ERROR, "Cannot Merge Self"); } @@ -257,7 +241,7 @@ ExecutionEngineImpl::Merge(const std::string &location) { double physical_size = server::CommonUtil::GetFileSize(location); server::CollectExecutionEngineMetrics metrics(physical_size); to_merge = read_index(location); - } catch (std::exception &e) { + } catch (std::exception& e) { ENGINE_LOG_ERROR << e.what(); return Status(DB_ERROR, e.what()); } @@ -280,7 +264,7 @@ ExecutionEngineImpl::Merge(const std::string &location) { } ExecutionEnginePtr -ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_type) { +ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_type) { ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_; auto from_index = std::dynamic_pointer_cast(index_); @@ -298,29 +282,23 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t temp_conf.gpu_id = gpu_num_; temp_conf.dim = Dimension(); temp_conf.nlist = nlist_; - temp_conf.metric_type = (metric_type_ == MetricType::IP) ? - knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2; + temp_conf.metric_type = (metric_type_ == MetricType::IP) ? knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2; temp_conf.size = Count(); auto adapter = AdapterMgr::GetInstance().GetAdapter(to_index->GetType()); auto conf = adapter->Match(temp_conf); - auto status = to_index->BuildAll(Count(), - from_index->GetRawVectors(), - from_index->GetRawIds(), - conf); - if (!status.ok()) { throw Exception(DB_ERROR, status.message()); } + auto status = to_index->BuildAll(Count(), from_index->GetRawVectors(), from_index->GetRawIds(), conf); + if (!status.ok()) { + throw Exception(DB_ERROR, status.message()); + } return std::make_shared(to_index, location, engine_type, metric_type_, nlist_); } Status -ExecutionEngineImpl::Search(int64_t n, - const float *data, - int64_t k, - int64_t nprobe, - float *distances, - int64_t *labels) const { +ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances, + int64_t* labels) const { if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search"; return Status(DB_ERROR, "index is null"); @@ -362,13 +340,13 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { // TODO(linxj): remove. Status ExecutionEngineImpl::Init() { - server::Config &config = server::Config::GetInstance(); + server::Config& config = server::Config::GetInstance(); Status s = config.GetDBConfigBuildIndexGPU(gpu_num_); if (!s.ok()) return s; return Status::OK(); } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/engine/ExecutionEngineImpl.h b/cpp/src/db/engine/ExecutionEngineImpl.h index 6ea09ddb29..e8f9521460 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.h +++ b/cpp/src/db/engine/ExecutionEngineImpl.h @@ -29,71 +29,81 @@ namespace engine { class ExecutionEngineImpl : public ExecutionEngine { public: - ExecutionEngineImpl(uint16_t dimension, - const std::string &location, - EngineType index_type, - MetricType metric_type, + ExecutionEngineImpl(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type, int32_t nlist); - ExecutionEngineImpl(VecIndexPtr index, - const std::string &location, - EngineType index_type, - MetricType metric_type, + ExecutionEngineImpl(VecIndexPtr index, const std::string& location, EngineType index_type, MetricType metric_type, int32_t nlist); - Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) override; + Status + AddWithIds(int64_t n, const float* xdata, const int64_t* xids) override; - size_t Count() const override; + size_t + Count() const override; - size_t Size() const override; + size_t + Size() const override; - size_t Dimension() const override; + size_t + Dimension() const override; - size_t PhysicalSize() const override; + size_t + PhysicalSize() const override; - Status Serialize() override; + Status + Serialize() override; - Status Load(bool to_cache) override; + Status + Load(bool to_cache) override; - Status CopyToGpu(uint64_t device_id) override; + Status + CopyToGpu(uint64_t device_id) override; - Status CopyToCpu() override; + Status + CopyToCpu() override; - ExecutionEnginePtr Clone() override; + ExecutionEnginePtr + Clone() override; - Status Merge(const std::string &location) override; + Status + Merge(const std::string& location) override; - Status Search(int64_t n, - const float *data, - int64_t k, - int64_t nprobe, - float *distances, - int64_t *labels) const override; + Status + Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances, int64_t* labels) const override; - ExecutionEnginePtr BuildIndex(const std::string &location, EngineType engine_type) override; + ExecutionEnginePtr + BuildIndex(const std::string& location, EngineType engine_type) override; - Status Cache() override; + Status + Cache() override; - Status GpuCache(uint64_t gpu_id) override; + Status + GpuCache(uint64_t gpu_id) override; - Status Init() override; + Status + Init() override; - EngineType IndexEngineType() const override { + EngineType + IndexEngineType() const override { return index_type_; } - MetricType IndexMetricType() const override { + MetricType + IndexMetricType() const override { return metric_type_; } - std::string GetLocation() const override { + std::string + GetLocation() const override { return location_; } private: - VecIndexPtr CreatetVecIndex(EngineType type); + VecIndexPtr + CreatetVecIndex(EngineType type); - VecIndexPtr Load(const std::string &location); + VecIndexPtr + Load(const std::string& location); protected: VecIndexPtr index_ = nullptr; @@ -107,6 +117,6 @@ class ExecutionEngineImpl : public ExecutionEngine { int32_t gpu_num_ = 0; }; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemManager.h b/cpp/src/db/insert/MemManager.h index 751633bf67..920e6e9692 100644 --- a/cpp/src/db/insert/MemManager.h +++ b/cpp/src/db/insert/MemManager.h @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "utils/Status.h" #include "db/Types.h" +#include "utils/Status.h" -#include #include +#include #include namespace zilliz { @@ -31,22 +30,27 @@ namespace engine { class MemManager { public: - virtual Status InsertVectors(const std::string &table_id, - size_t n, const float *vectors, IDNumbers &vector_ids) = 0; + virtual Status + InsertVectors(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids) = 0; - virtual Status Serialize(std::set &table_ids) = 0; + virtual Status + Serialize(std::set& table_ids) = 0; - virtual Status EraseMemVector(const std::string &table_id) = 0; + virtual Status + EraseMemVector(const std::string& table_id) = 0; - virtual size_t GetCurrentMutableMem() = 0; + virtual size_t + GetCurrentMutableMem() = 0; - virtual size_t GetCurrentImmutableMem() = 0; + virtual size_t + GetCurrentImmutableMem() = 0; - virtual size_t GetCurrentMem() = 0; -}; // MemManagerAbstract + virtual size_t + GetCurrentMem() = 0; +}; // MemManagerAbstract using MemManagerPtr = std::shared_ptr; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemManagerImpl.cpp b/cpp/src/db/insert/MemManagerImpl.cpp index e555acd28d..1a59c8b892 100644 --- a/cpp/src/db/insert/MemManagerImpl.cpp +++ b/cpp/src/db/insert/MemManagerImpl.cpp @@ -15,11 +15,10 @@ // specific language governing permissions and limitations // under the License. - #include "db/insert/MemManagerImpl.h" #include "VectorSource.h" -#include "utils/Log.h" #include "db/Constants.h" +#include "utils/Log.h" #include @@ -28,7 +27,7 @@ namespace milvus { namespace engine { MemTablePtr -MemManagerImpl::GetMemByTable(const std::string &table_id) { +MemManagerImpl::GetMemByTable(const std::string& table_id) { auto memIt = mem_id_map_.find(table_id); if (memIt != mem_id_map_.end()) { return memIt->second; @@ -39,10 +38,7 @@ MemManagerImpl::GetMemByTable(const std::string &table_id) { } Status -MemManagerImpl::InsertVectors(const std::string &table_id_, - size_t n_, - const float *vectors_, - IDNumbers &vector_ids_) { +MemManagerImpl::InsertVectors(const std::string& table_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_) { while (GetCurrentMem() > options_.insert_buffer_size_) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); } @@ -53,10 +49,8 @@ MemManagerImpl::InsertVectors(const std::string &table_id_, } Status -MemManagerImpl::InsertVectorsNoLock(const std::string &table_id, - size_t n, - const float *vectors, - IDNumbers &vector_ids) { +MemManagerImpl::InsertVectorsNoLock(const std::string& table_id, size_t n, const float* vectors, + IDNumbers& vector_ids) { MemTablePtr mem = GetMemByTable(table_id); VectorSourcePtr source = std::make_shared(n, vectors); @@ -73,9 +67,9 @@ Status MemManagerImpl::ToImmutable() { std::unique_lock lock(mutex_); MemIdMap temp_map; - for (auto &kv : mem_id_map_) { + for (auto& kv : mem_id_map_) { if (kv.second->Empty()) { - //empty table, no need to serialize + // empty table, no need to serialize temp_map.insert(kv); } else { immu_mem_list_.push_back(kv.second); @@ -87,11 +81,11 @@ MemManagerImpl::ToImmutable() { } Status -MemManagerImpl::Serialize(std::set &table_ids) { +MemManagerImpl::Serialize(std::set& table_ids) { ToImmutable(); std::unique_lock lock(serialization_mtx_); table_ids.clear(); - for (auto &mem : immu_mem_list_) { + for (auto& mem : immu_mem_list_) { mem->Serialize(); table_ids.insert(mem->GetTableId()); } @@ -100,16 +94,16 @@ MemManagerImpl::Serialize(std::set &table_ids) { } Status -MemManagerImpl::EraseMemVector(const std::string &table_id) { - {//erase MemVector from rapid-insert cache +MemManagerImpl::EraseMemVector(const std::string& table_id) { + { // erase MemVector from rapid-insert cache std::unique_lock lock(mutex_); mem_id_map_.erase(table_id); } - {//erase MemVector from serialize cache + { // erase MemVector from serialize cache std::unique_lock lock(serialization_mtx_); MemList temp_list; - for (auto &mem : immu_mem_list_) { + for (auto& mem : immu_mem_list_) { if (mem->GetTableId() != table_id) { temp_list.push_back(mem); } @@ -123,7 +117,7 @@ MemManagerImpl::EraseMemVector(const std::string &table_id) { size_t MemManagerImpl::GetCurrentMutableMem() { size_t total_mem = 0; - for (auto &kv : mem_id_map_) { + for (auto& kv : mem_id_map_) { auto memTable = kv.second; total_mem += memTable->GetCurrentMem(); } @@ -133,7 +127,7 @@ MemManagerImpl::GetCurrentMutableMem() { size_t MemManagerImpl::GetCurrentImmutableMem() { size_t total_mem = 0; - for (auto &mem_table : immu_mem_list_) { + for (auto& mem_table : immu_mem_list_) { total_mem += mem_table->GetCurrentMem(); } return total_mem; @@ -144,6 +138,6 @@ MemManagerImpl::GetCurrentMem() { return GetCurrentMutableMem() + GetCurrentImmutableMem(); } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemManagerImpl.h b/cpp/src/db/insert/MemManagerImpl.h index 1783adec18..4de5626459 100644 --- a/cpp/src/db/insert/MemManagerImpl.h +++ b/cpp/src/db/insert/MemManagerImpl.h @@ -15,21 +15,20 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "MemTable.h" #include "MemManager.h" +#include "MemTable.h" #include "db/meta/Meta.h" #include "utils/Status.h" -#include -#include -#include -#include #include +#include #include #include +#include +#include +#include namespace zilliz { namespace milvus { @@ -39,29 +38,35 @@ class MemManagerImpl : public MemManager { public: using Ptr = std::shared_ptr; - MemManagerImpl(const meta::MetaPtr &meta, const DBOptions &options) - : meta_(meta), options_(options) { + MemManagerImpl(const meta::MetaPtr& meta, const DBOptions& options) : meta_(meta), options_(options) { } - Status InsertVectors(const std::string &table_id, - size_t n, const float *vectors, IDNumbers &vector_ids) override; + Status + InsertVectors(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids) override; - Status Serialize(std::set &table_ids) override; + Status + Serialize(std::set& table_ids) override; - Status EraseMemVector(const std::string &table_id) override; + Status + EraseMemVector(const std::string& table_id) override; - size_t GetCurrentMutableMem() override; + size_t + GetCurrentMutableMem() override; - size_t GetCurrentImmutableMem() override; + size_t + GetCurrentImmutableMem() override; - size_t GetCurrentMem() override; + size_t + GetCurrentMem() override; private: - MemTablePtr GetMemByTable(const std::string &table_id); + MemTablePtr + GetMemByTable(const std::string& table_id); - Status InsertVectorsNoLock(const std::string &table_id, - size_t n, const float *vectors, IDNumbers &vector_ids); - Status ToImmutable(); + Status + InsertVectorsNoLock(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids); + Status + ToImmutable(); using MemIdMap = std::map; using MemList = std::vector; @@ -71,8 +76,8 @@ class MemManagerImpl : public MemManager { DBOptions options_; std::mutex mutex_; std::mutex serialization_mtx_; -}; // NewMemManager +}; // NewMemManager -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemMenagerFactory.cpp b/cpp/src/db/insert/MemMenagerFactory.cpp index 76e521e168..8f0d6704d7 100644 --- a/cpp/src/db/insert/MemMenagerFactory.cpp +++ b/cpp/src/db/insert/MemMenagerFactory.cpp @@ -17,26 +17,26 @@ #include "db/insert/MemMenagerFactory.h" #include "MemManagerImpl.h" -#include "utils/Log.h" #include "utils/Exception.h" +#include "utils/Log.h" #include #include -#include #include -#include -#include #include +#include +#include +#include namespace zilliz { namespace milvus { namespace engine { MemManagerPtr -MemManagerFactory::Build(const std::shared_ptr &meta, const DBOptions &options) { +MemManagerFactory::Build(const std::shared_ptr& meta, const DBOptions& options) { return std::make_shared(meta, options); } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemMenagerFactory.h b/cpp/src/db/insert/MemMenagerFactory.h index fa0cb5930a..d7996abe99 100644 --- a/cpp/src/db/insert/MemMenagerFactory.h +++ b/cpp/src/db/insert/MemMenagerFactory.h @@ -28,9 +28,10 @@ namespace engine { class MemManagerFactory { public: - static MemManagerPtr Build(const std::shared_ptr &meta, const DBOptions &options); + static MemManagerPtr + Build(const std::shared_ptr& meta, const DBOptions& options); }; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTable.cpp b/cpp/src/db/insert/MemTable.cpp index 8db6a9d141..e189dbf15b 100644 --- a/cpp/src/db/insert/MemTable.cpp +++ b/cpp/src/db/insert/MemTable.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "db/insert/MemTable.h" #include "utils/Log.h" @@ -26,16 +25,12 @@ namespace zilliz { namespace milvus { namespace engine { -MemTable::MemTable(const std::string &table_id, - const meta::MetaPtr &meta, - const DBOptions &options) : - table_id_(table_id), - meta_(meta), - options_(options) { +MemTable::MemTable(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options) + : table_id_(table_id), meta_(meta), options_(options) { } Status -MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) { +MemTable::Add(VectorSourcePtr& source, IDNumbers& vector_ids) { while (!source->AllAdded()) { MemTableFilePtr current_mem_table_file; if (!mem_table_file_list_.empty()) { @@ -63,7 +58,7 @@ MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) { } void -MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) { +MemTable::GetCurrentMemTableFile(MemTableFilePtr& mem_table_file) { mem_table_file = mem_table_file_list_.back(); } @@ -92,7 +87,7 @@ MemTable::Empty() { return mem_table_file_list_.empty(); } -const std::string & +const std::string& MemTable::GetTableId() const { return table_id_; } @@ -101,12 +96,12 @@ size_t MemTable::GetCurrentMem() { std::lock_guard lock(mutex_); size_t total_mem = 0; - for (auto &mem_table_file : mem_table_file_list_) { + for (auto& mem_table_file : mem_table_file_list_) { total_mem += mem_table_file->GetCurrentMem(); } return total_mem; } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTable.h b/cpp/src/db/insert/MemTable.h index da7d914b41..b2b16cf472 100644 --- a/cpp/src/db/insert/MemTable.h +++ b/cpp/src/db/insert/MemTable.h @@ -15,17 +15,16 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "MemTableFile.h" #include "VectorSource.h" #include "utils/Status.h" -#include -#include #include +#include #include +#include namespace zilliz { namespace milvus { @@ -35,21 +34,28 @@ class MemTable { public: using MemTableFileList = std::vector; - MemTable(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options); + MemTable(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options); - Status Add(VectorSourcePtr &source, IDNumbers &vector_ids); + Status + Add(VectorSourcePtr& source, IDNumbers& vector_ids); - void GetCurrentMemTableFile(MemTableFilePtr &mem_table_file); + void + GetCurrentMemTableFile(MemTableFilePtr& mem_table_file); - size_t GetTableFileCount(); + size_t + GetTableFileCount(); - Status Serialize(); + Status + Serialize(); - bool Empty(); + bool + Empty(); - const std::string &GetTableId() const; + const std::string& + GetTableId() const; - size_t GetCurrentMem(); + size_t + GetCurrentMem(); private: const std::string table_id_; @@ -61,10 +67,10 @@ class MemTable { DBOptions options_; std::mutex mutex_; -}; //MemTable +}; // MemTable using MemTablePtr = std::shared_ptr; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index fc6c2b319a..8395b0b62a 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "db/insert/MemTableFile.h" #include "db/Constants.h" #include "db/engine/EngineFactory.h" @@ -29,20 +28,14 @@ namespace zilliz { namespace milvus { namespace engine { -MemTableFile::MemTableFile(const std::string &table_id, - const meta::MetaPtr &meta, - const DBOptions &options) : - table_id_(table_id), - meta_(meta), - options_(options) { +MemTableFile::MemTableFile(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options) + : table_id_(table_id), meta_(meta), options_(options) { current_mem_ = 0; auto status = CreateTableFile(); if (status.ok()) { - execution_engine_ = EngineFactory::Build(table_file_schema_.dimension_, - table_file_schema_.location_, - (EngineType) table_file_schema_.engine_type_, - (MetricType) table_file_schema_.metric_type_, - table_file_schema_.nlist_); + execution_engine_ = EngineFactory::Build( + table_file_schema_.dimension_, table_file_schema_.location_, (EngineType)table_file_schema_.engine_type_, + (MetricType)table_file_schema_.metric_type_, table_file_schema_.nlist_); } } @@ -61,10 +54,11 @@ MemTableFile::CreateTableFile() { } Status -MemTableFile::Add(const VectorSourcePtr &source, IDNumbers &vector_ids) { +MemTableFile::Add(const VectorSourcePtr& source, IDNumbers& vector_ids) { if (table_file_schema_.dimension_ <= 0) { std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + - std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_; + std::to_string(table_file_schema_.dimension_) + ", table_id = " + + table_file_schema_.table_id_; ENGINE_LOG_ERROR << err_msg; return Status(DB_ERROR, "Not able to create table file"); } @@ -109,11 +103,11 @@ MemTableFile::Serialize() { table_file_schema_.file_size_ = execution_engine_->PhysicalSize(); table_file_schema_.row_count_ = execution_engine_->Count(); - //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size - //else set file type to RAW, no need to build index - if (table_file_schema_.engine_type_ != (int) EngineType::FAISS_IDMAP) { - table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? - meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + // if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size + // else set file type to RAW, no need to build index + if (table_file_schema_.engine_type_ != (int)EngineType::FAISS_IDMAP) { + table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? meta::TableFileSchema::TO_INDEX + : meta::TableFileSchema::RAW; } else { table_file_schema_.file_type_ = meta::TableFileSchema::RAW; } @@ -130,6 +124,6 @@ MemTableFile::Serialize() { return status; } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTableFile.h b/cpp/src/db/insert/MemTableFile.h index 1d7019a42d..72ae09c861 100644 --- a/cpp/src/db/insert/MemTableFile.h +++ b/cpp/src/db/insert/MemTableFile.h @@ -15,16 +15,15 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "VectorSource.h" -#include "db/meta/Meta.h" #include "db/engine/ExecutionEngine.h" +#include "db/meta/Meta.h" #include "utils/Status.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -32,20 +31,26 @@ namespace engine { class MemTableFile { public: - MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options); + MemTableFile(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options); - Status Add(const VectorSourcePtr &source, IDNumbers &vector_ids); + Status + Add(const VectorSourcePtr& source, IDNumbers& vector_ids); - size_t GetCurrentMem(); + size_t + GetCurrentMem(); - size_t GetMemLeft(); + size_t + GetMemLeft(); - bool IsFull(); + bool + IsFull(); - Status Serialize(); + Status + Serialize(); private: - Status CreateTableFile(); + Status + CreateTableFile(); private: const std::string table_id_; @@ -55,10 +60,10 @@ class MemTableFile { size_t current_mem_; ExecutionEnginePtr execution_engine_; -}; //MemTableFile +}; // MemTableFile using MemTableFilePtr = std::shared_ptr; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/VectorSource.cpp b/cpp/src/db/insert/VectorSource.cpp index 330ef08b98..7a9fdf5d0f 100644 --- a/cpp/src/db/insert/VectorSource.cpp +++ b/cpp/src/db/insert/VectorSource.cpp @@ -15,35 +15,28 @@ // specific language governing permissions and limitations // under the License. - #include "db/insert/VectorSource.h" -#include "db/engine/ExecutionEngine.h" #include "db/engine/EngineFactory.h" -#include "utils/Log.h" +#include "db/engine/ExecutionEngine.h" #include "metrics/Metrics.h" +#include "utils/Log.h" namespace zilliz { namespace milvus { namespace engine { -VectorSource::VectorSource(const size_t &n, - const float *vectors) : - n_(n), - vectors_(vectors), - id_generator_(std::make_shared()) { +VectorSource::VectorSource(const size_t& n, const float* vectors) + : n_(n), vectors_(vectors), id_generator_(std::make_shared()) { current_num_vectors_added = 0; } Status -VectorSource::Add(const ExecutionEnginePtr &execution_engine, - const meta::TableFileSchema &table_file_schema, - const size_t &num_vectors_to_add, - size_t &num_vectors_added, - IDNumbers &vector_ids) { +VectorSource::Add(const ExecutionEnginePtr& execution_engine, const meta::TableFileSchema& table_file_schema, + const size_t& num_vectors_to_add, size_t& num_vectors_added, IDNumbers& vector_ids) { server::CollectAddMetrics metrics(n_, table_file_schema.dimension_); - num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ? - num_vectors_to_add : n_ - current_num_vectors_added; + num_vectors_added = + current_num_vectors_added + num_vectors_to_add <= n_ ? num_vectors_to_add : n_ - current_num_vectors_added; IDNumbers vector_ids_to_add; if (vector_ids.empty()) { id_generator_->GetNextIDNumbers(num_vectors_added, vector_ids_to_add); @@ -58,8 +51,7 @@ VectorSource::Add(const ExecutionEnginePtr &execution_engine, vector_ids_to_add.data()); if (status.ok()) { current_num_vectors_added += num_vectors_added; - vector_ids_.insert(vector_ids_.end(), - std::make_move_iterator(vector_ids_to_add.begin()), + vector_ids_.insert(vector_ids_.end(), std::make_move_iterator(vector_ids_to_add.begin()), std::make_move_iterator(vector_ids_to_add.end())); } else { ENGINE_LOG_ERROR << "VectorSource::Add failed: " + status.ToString(); @@ -83,6 +75,6 @@ VectorSource::GetVectorIds() { return vector_ids_; } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/VectorSource.h b/cpp/src/db/insert/VectorSource.h index fd31a14fa6..4b8e9d21b9 100644 --- a/cpp/src/db/insert/VectorSource.h +++ b/cpp/src/db/insert/VectorSource.h @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "db/meta/Meta.h" #include "db/IDGenerator.h" #include "db/engine/ExecutionEngine.h" +#include "db/meta/Meta.h" #include "utils/Status.h" #include @@ -31,32 +30,33 @@ namespace engine { class VectorSource { public: - VectorSource(const size_t &n, const float *vectors); + VectorSource(const size_t& n, const float* vectors); - Status Add(const ExecutionEnginePtr &execution_engine, - const meta::TableFileSchema &table_file_schema, - const size_t &num_vectors_to_add, - size_t &num_vectors_added, - IDNumbers &vector_ids); + Status + Add(const ExecutionEnginePtr& execution_engine, const meta::TableFileSchema& table_file_schema, + const size_t& num_vectors_to_add, size_t& num_vectors_added, IDNumbers& vector_ids); - size_t GetNumVectorsAdded(); + size_t + GetNumVectorsAdded(); - bool AllAdded(); + bool + AllAdded(); - IDNumbers GetVectorIds(); + IDNumbers + GetVectorIds(); private: const size_t n_; - const float *vectors_; + const float* vectors_; IDNumbers vector_ids_; size_t current_num_vectors_added; std::shared_ptr id_generator_; -}; //VectorSource +}; // VectorSource using VectorSourcePtr = std::shared_ptr; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index 42fe6a6dd5..3e3128366c 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "MetaTypes.h" @@ -25,84 +24,104 @@ #include #include -#include #include +#include namespace zilliz { namespace milvus { namespace engine { namespace meta { -static const char *META_TABLES = "Tables"; -static const char *META_TABLEFILES = "TableFiles"; +static const char* META_TABLES = "Tables"; +static const char* META_TABLEFILES = "TableFiles"; class Meta { public: virtual ~Meta() = default; - virtual Status CreateTable(TableSchema &table_schema) = 0; + virtual Status + CreateTable(TableSchema& table_schema) = 0; - virtual Status DescribeTable(TableSchema &table_schema) = 0; + virtual Status + DescribeTable(TableSchema& table_schema) = 0; - virtual Status HasTable(const std::string &table_id, bool &has_or_not) = 0; + virtual Status + HasTable(const std::string& table_id, bool& has_or_not) = 0; - virtual Status AllTables(std::vector &table_schema_array) = 0; + virtual Status + AllTables(std::vector& table_schema_array) = 0; - virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) = 0; + virtual Status + UpdateTableIndex(const std::string& table_id, const TableIndex& index) = 0; - virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0; + virtual Status + UpdateTableFlag(const std::string& table_id, int64_t flag) = 0; - virtual Status DeleteTable(const std::string &table_id) = 0; + virtual Status + DeleteTable(const std::string& table_id) = 0; - virtual Status DeleteTableFiles(const std::string &table_id) = 0; + virtual Status + DeleteTableFiles(const std::string& table_id) = 0; - virtual Status CreateTableFile(TableFileSchema &file_schema) = 0; + virtual Status + CreateTableFile(TableFileSchema& file_schema) = 0; - virtual Status DropPartitionsByDates(const std::string &table_id, const DatesT &dates) = 0; + virtual Status + DropPartitionsByDates(const std::string& table_id, const DatesT& dates) = 0; - virtual Status GetTableFiles(const std::string &table_id, - const std::vector &ids, - TableFilesSchema &table_files) = 0; + virtual Status + GetTableFiles(const std::string& table_id, const std::vector& ids, TableFilesSchema& table_files) = 0; - virtual Status UpdateTableFilesToIndex(const std::string &table_id) = 0; + virtual Status + UpdateTableFilesToIndex(const std::string& table_id) = 0; - virtual Status UpdateTableFile(TableFileSchema &file_schema) = 0; + virtual Status + UpdateTableFile(TableFileSchema& file_schema) = 0; - virtual Status UpdateTableFiles(TableFilesSchema &files) = 0; + virtual Status + UpdateTableFiles(TableFilesSchema& files) = 0; - virtual Status FilesToSearch(const std::string &table_id, - const std::vector &ids, - const DatesT &partition, - DatePartionedTableFilesSchema &files) = 0; + virtual Status + FilesToSearch(const std::string& table_id, const std::vector& ids, const DatesT& partition, + DatePartionedTableFilesSchema& files) = 0; - virtual Status FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) = 0; + virtual Status + FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) = 0; - virtual Status Size(uint64_t &result) = 0; + virtual Status + Size(uint64_t& result) = 0; - virtual Status Archive() = 0; + virtual Status + Archive() = 0; - virtual Status FilesToIndex(TableFilesSchema &) = 0; + virtual Status + FilesToIndex(TableFilesSchema&) = 0; - virtual Status FilesByType(const std::string &table_id, - const std::vector &file_types, - std::vector &file_ids) = 0; + virtual Status + FilesByType(const std::string& table_id, const std::vector& file_types, + std::vector& file_ids) = 0; - virtual Status DescribeTableIndex(const std::string &table_id, TableIndex &index) = 0; + virtual Status + DescribeTableIndex(const std::string& table_id, TableIndex& index) = 0; - virtual Status DropTableIndex(const std::string &table_id) = 0; + virtual Status + DropTableIndex(const std::string& table_id) = 0; - virtual Status CleanUp() = 0; + virtual Status + CleanUp() = 0; virtual Status CleanUpFilesWithTTL(uint16_t) = 0; - virtual Status DropAll() = 0; + virtual Status + DropAll() = 0; - virtual Status Count(const std::string &table_id, uint64_t &result) = 0; -}; // MetaData + virtual Status + Count(const std::string& table_id, uint64_t& result) = 0; +}; // MetaData using MetaPtr = std::shared_ptr; -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MetaConsts.h b/cpp/src/db/meta/MetaConsts.h index 6abae564e8..64cfeab78a 100644 --- a/cpp/src/db/meta/MetaConsts.h +++ b/cpp/src/db/meta/MetaConsts.h @@ -38,7 +38,7 @@ const size_t H_SEC = 60 * M_SEC; const size_t D_SEC = 24 * H_SEC; const size_t W_SEC = 7 * D_SEC; -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MetaFactory.cpp b/cpp/src/db/meta/MetaFactory.cpp index 6a305a4280..d8beecbd4e 100644 --- a/cpp/src/db/meta/MetaFactory.cpp +++ b/cpp/src/db/meta/MetaFactory.cpp @@ -16,26 +16,26 @@ // under the License. #include "db/meta/MetaFactory.h" -#include "SqliteMetaImpl.h" #include "MySQLMetaImpl.h" -#include "utils/Log.h" -#include "utils/Exception.h" +#include "SqliteMetaImpl.h" #include "db/Utils.h" +#include "utils/Exception.h" +#include "utils/Log.h" #include -#include -#include -#include -#include #include +#include +#include #include +#include +#include namespace zilliz { namespace milvus { namespace engine { DBMetaOptions -MetaFactory::BuildOption(const std::string &path) { +MetaFactory::BuildOption(const std::string& path) { auto p = path; if (p == "") { srand(time(nullptr)); @@ -51,7 +51,7 @@ MetaFactory::BuildOption(const std::string &path) { } meta::MetaPtr -MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) { +MetaFactory::Build(const DBMetaOptions& metaOptions, const int& mode) { std::string uri = metaOptions.backend_uri_; utils::MetaUriInfo uri_info; @@ -73,6 +73,6 @@ MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) { } } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MetaFactory.h b/cpp/src/db/meta/MetaFactory.h index a13a09715a..0792515ac1 100644 --- a/cpp/src/db/meta/MetaFactory.h +++ b/cpp/src/db/meta/MetaFactory.h @@ -28,11 +28,13 @@ namespace engine { class MetaFactory { public: - static DBMetaOptions BuildOption(const std::string &path = ""); + static DBMetaOptions + BuildOption(const std::string& path = ""); - static meta::MetaPtr Build(const DBMetaOptions &metaOptions, const int &mode); + static meta::MetaPtr + Build(const DBMetaOptions& metaOptions, const int& mode); }; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h index 6fd649bc07..aad8b61c87 100644 --- a/cpp/src/db/meta/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -17,22 +17,22 @@ #pragma once -#include "db/engine/ExecutionEngine.h" #include "db/Constants.h" +#include "db/engine/ExecutionEngine.h" -#include #include -#include #include +#include +#include namespace zilliz { namespace milvus { namespace engine { namespace meta { -constexpr int32_t DEFAULT_ENGINE_TYPE = (int) EngineType::FAISS_IDMAP; +constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP; constexpr int32_t DEFAULT_NLIST = 16384; -constexpr int32_t DEFAULT_METRIC_TYPE = (int) MetricType::L2; +constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2; constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB; constexpr int64_t FLAG_MASK_NO_USERID = 0x1; @@ -50,7 +50,7 @@ struct TableSchema { size_t id_ = 0; std::string table_id_; - int32_t state_ = (int) NORMAL; + int32_t state_ = (int)NORMAL; uint16_t dimension_ = 0; int64_t created_on_ = 0; int64_t flag_ = 0; @@ -58,7 +58,7 @@ struct TableSchema { int32_t engine_type_ = DEFAULT_ENGINE_TYPE; int32_t nlist_ = DEFAULT_NLIST; int32_t metric_type_ = DEFAULT_METRIC_TYPE; -}; // TableSchema +}; // TableSchema struct TableFileSchema { typedef enum { @@ -83,17 +83,17 @@ struct TableFileSchema { std::string location_; int64_t updated_time_ = 0; int64_t created_on_ = 0; - int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; //not persist to meta + int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; // not persist to meta int32_t engine_type_ = DEFAULT_ENGINE_TYPE; - int32_t nlist_ = DEFAULT_NLIST; //not persist to meta - int32_t metric_type_ = DEFAULT_METRIC_TYPE; //not persist to meta -}; // TableFileSchema + int32_t nlist_ = DEFAULT_NLIST; // not persist to meta + int32_t metric_type_ = DEFAULT_METRIC_TYPE; // not persist to meta +}; // TableFileSchema using TableFileSchemaPtr = std::shared_ptr; using TableFilesSchema = std::vector; using DatePartionedTableFilesSchema = std::map; -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MySQLConnectionPool.cpp b/cpp/src/db/meta/MySQLConnectionPool.cpp index 9d612d8b54..2d09874d3f 100644 --- a/cpp/src/db/meta/MySQLConnectionPool.cpp +++ b/cpp/src/db/meta/MySQLConnectionPool.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "db/meta/MySQLConnectionPool.h" namespace zilliz { @@ -28,7 +27,7 @@ namespace meta { // already. Can't do this in create() because we're interested in // connections actually in use, not those created. Also note that // we keep our own count; ConnectionPool::size() isn't the same! -mysqlpp::Connection * +mysqlpp::Connection* MySQLConnectionPool::grab() { while (conns_in_use_ > max_pool_size_) { sleep(1); @@ -40,7 +39,7 @@ MySQLConnectionPool::grab() { // Other half of in-use conn count limit void -MySQLConnectionPool::release(const mysqlpp::Connection *pc) { +MySQLConnectionPool::release(const mysqlpp::Connection* pc) { mysqlpp::ConnectionPool::release(pc); if (conns_in_use_ <= 0) { ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " @@ -64,27 +63,25 @@ MySQLConnectionPool::getDB() { } // Superclass overrides -mysqlpp::Connection * +mysqlpp::Connection* MySQLConnectionPool::create() { try { // Create connection using the parameters we were passed upon // creation. - mysqlpp::Connection *conn = new mysqlpp::Connection(); + mysqlpp::Connection* conn = new mysqlpp::Connection(); conn->set_option(new mysqlpp::ReconnectOption(true)); - conn->connect(db_.empty() ? 0 : db_.c_str(), - server_.empty() ? 0 : server_.c_str(), - user_.empty() ? 0 : user_.c_str(), - password_.empty() ? 0 : password_.c_str(), - port_); + conn->connect(db_.empty() ? 0 : db_.c_str(), server_.empty() ? 0 : server_.c_str(), + user_.empty() ? 0 : user_.c_str(), password_.empty() ? 0 : password_.c_str(), port_); return conn; - } catch (const mysqlpp::ConnectionFailed &er) { - ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what(); + } catch (const mysqlpp::ConnectionFailed& er) { + ENGINE_LOG_ERROR << "Failed to connect to database server" + << ": " << er.what(); return nullptr; } } void -MySQLConnectionPool::destroy(mysqlpp::Connection *cp) { +MySQLConnectionPool::destroy(mysqlpp::Connection* cp) { // Our superclass can't know how we created the Connection, so // it delegates destruction to us, to be safe. delete cp; @@ -95,7 +92,7 @@ MySQLConnectionPool::max_idle_time() { return max_idle_time_; } -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MySQLConnectionPool.h b/cpp/src/db/meta/MySQLConnectionPool.h index 761f272bc1..1b5345c762 100644 --- a/cpp/src/db/meta/MySQLConnectionPool.h +++ b/cpp/src/db/meta/MySQLConnectionPool.h @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. - #include "mysql++/mysql++.h" -#include #include #include +#include #include "utils/Log.h" @@ -32,20 +31,16 @@ namespace meta { class MySQLConnectionPool : public mysqlpp::ConnectionPool { public: // The object's only constructor - MySQLConnectionPool(std::string dbName, - std::string userName, - std::string passWord, - std::string serverIp, - int port = 0, - int maxPoolSize = 8) : - db_(dbName), - user_(userName), - password_(passWord), - server_(serverIp), - port_(port), - max_pool_size_(maxPoolSize) { + MySQLConnectionPool(std::string dbName, std::string userName, std::string passWord, std::string serverIp, + int port = 0, int maxPoolSize = 8) + : db_(dbName), + user_(userName), + password_(passWord), + server_(serverIp), + port_(port), + max_pool_size_(maxPoolSize) { conns_in_use_ = 0; - max_idle_time_ = 10; //10 seconds + max_idle_time_ = 10; // 10 seconds } // The destructor. We _must_ call ConnectionPool::clear() here, @@ -54,24 +49,30 @@ class MySQLConnectionPool : public mysqlpp::ConnectionPool { clear(); } - mysqlpp::Connection *grab() override; + mysqlpp::Connection* + grab() override; // Other half of in-use conn count limit - void release(const mysqlpp::Connection *pc) override; + void + release(const mysqlpp::Connection* pc) override; -// int getConnectionsInUse(); -// -// void set_max_idle_time(int max_idle); + // int getConnectionsInUse(); + // + // void set_max_idle_time(int max_idle); - std::string getDB(); + std::string + getDB(); protected: // Superclass overrides - mysqlpp::Connection *create() override; + mysqlpp::Connection* + create() override; - void destroy(mysqlpp::Connection *cp) override; + void + destroy(mysqlpp::Connection* cp) override; - unsigned int max_idle_time() override; + unsigned int + max_idle_time() override; private: // Number of connections currently in use @@ -86,7 +87,7 @@ class MySQLConnectionPool : public mysqlpp::ConnectionPool { unsigned int max_idle_time_; }; -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 490eda1916..e9654c7866 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -16,27 +16,27 @@ // under the License. #include "db/meta/MySQLMetaImpl.h" +#include "MetaConsts.h" #include "db/IDGenerator.h" #include "db/Utils.h" -#include "utils/Log.h" -#include "utils/Exception.h" -#include "MetaConsts.h" #include "metrics/Metrics.h" +#include "utils/Exception.h" +#include "utils/Log.h" +#include +#include #include -#include -#include +#include #include #include -#include -#include -#include -#include -#include +#include #include +#include +#include #include -#include -#include +#include +#include +#include namespace zilliz { namespace milvus { @@ -46,7 +46,7 @@ namespace meta { namespace { Status -HandleException(const std::string &desc, const char *what = nullptr) { +HandleException(const std::string& desc, const char* what = nullptr) { if (what == nullptr) { ENGINE_LOG_ERROR << desc; return Status(DB_META_TRANSACTION_FAILED, desc); @@ -59,27 +59,28 @@ HandleException(const std::string &desc, const char *what = nullptr) { class MetaField { public: - MetaField(const std::string &name, const std::string &type, const std::string &setting) - : name_(name), - type_(type), - setting_(setting) { + MetaField(const std::string& name, const std::string& type, const std::string& setting) + : name_(name), type_(type), setting_(setting) { } - std::string name() const { + std::string + name() const { return name_; } - std::string ToString() const { + std::string + ToString() const { return name_ + " " + type_ + " " + setting_; } // mysql field type has additional information. for instance, a filed type is defined as 'BIGINT' // we get the type from sql is 'bigint(20)', so we need to ignore the '(20)' - bool IsEqual(const MetaField &field) const { + bool + IsEqual(const MetaField& field) const { size_t name_len_min = field.name_.length() > name_.length() ? name_.length() : field.name_.length(); size_t type_len_min = field.type_.length() > type_.length() ? type_.length() : field.type_.length(); return strncasecmp(field.name_.c_str(), name_.c_str(), name_len_min) == 0 && - strncasecmp(field.type_.c_str(), type_.c_str(), type_len_min) == 0; + strncasecmp(field.type_.c_str(), type_.c_str(), type_len_min) == 0; } private: @@ -91,18 +92,18 @@ class MetaField { using MetaFields = std::vector; class MetaSchema { public: - MetaSchema(const std::string &name, const MetaFields &fields) - : name_(name), - fields_(fields) { + MetaSchema(const std::string& name, const MetaFields& fields) : name_(name), fields_(fields) { } - std::string name() const { + std::string + name() const { return name_; } - std::string ToString() const { + std::string + ToString() const { std::string result; - for (auto &field : fields_) { + for (auto& field : fields_) { if (!result.empty()) { result += ","; } @@ -111,12 +112,13 @@ class MetaSchema { return result; } - //if the outer fields contains all this MetaSchema fields, return true - //otherwise return false - bool IsEqual(const MetaFields &fields) const { + // if the outer fields contains all this MetaSchema fields, return true + // otherwise return false + bool + IsEqual(const MetaFields& fields) const { std::vector found_field; - for (const auto &this_field : fields_) { - for (const auto &outer_field : fields) { + for (const auto& this_field : fields_) { + for (const auto& outer_field : fields) { if (this_field.IsEqual(outer_field)) { found_field.push_back(this_field.name()); break; @@ -132,40 +134,35 @@ class MetaSchema { MetaFields fields_; }; -//Tables schema +// Tables schema static const MetaSchema TABLES_SCHEMA(META_TABLES, { - MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), - MetaField("table_id", "VARCHAR(255)", "UNIQUE NOT NULL"), - MetaField("state", "INT", "NOT NULL"), - MetaField("dimension", "SMALLINT", "NOT NULL"), - MetaField("created_on", "BIGINT", "NOT NULL"), - MetaField("flag", "BIGINT", "DEFAULT 0 NOT NULL"), - MetaField("index_file_size", "BIGINT", "DEFAULT 1024 NOT NULL"), - MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), - MetaField("nlist", "INT", "DEFAULT 16384 NOT NULL"), - MetaField("metric_type", "INT", "DEFAULT 1 NOT NULL"), -}); + MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), + MetaField("table_id", "VARCHAR(255)", "UNIQUE NOT NULL"), + MetaField("state", "INT", "NOT NULL"), + MetaField("dimension", "SMALLINT", "NOT NULL"), + MetaField("created_on", "BIGINT", "NOT NULL"), + MetaField("flag", "BIGINT", "DEFAULT 0 NOT NULL"), + MetaField("index_file_size", "BIGINT", "DEFAULT 1024 NOT NULL"), + MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), + MetaField("nlist", "INT", "DEFAULT 16384 NOT NULL"), + MetaField("metric_type", "INT", "DEFAULT 1 NOT NULL"), + }); -//TableFiles schema -static const MetaSchema TABLEFILES_SCHEMA(META_TABLEFILES, { - MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), - MetaField("table_id", "VARCHAR(255)", "NOT NULL"), - MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), - MetaField("file_id", "VARCHAR(255)", "NOT NULL"), - MetaField("file_type", "INT", "DEFAULT 0 NOT NULL"), - MetaField("file_size", "BIGINT", "DEFAULT 0 NOT NULL"), - MetaField("row_count", "BIGINT", "DEFAULT 0 NOT NULL"), - MetaField("updated_time", "BIGINT", "NOT NULL"), - MetaField("created_on", "BIGINT", "NOT NULL"), - MetaField("date", "INT", "DEFAULT -1 NOT NULL"), -}); +// TableFiles schema +static const MetaSchema TABLEFILES_SCHEMA( + META_TABLEFILES, + { + MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), MetaField("table_id", "VARCHAR(255)", "NOT NULL"), + MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), MetaField("file_id", "VARCHAR(255)", "NOT NULL"), + MetaField("file_type", "INT", "DEFAULT 0 NOT NULL"), MetaField("file_size", "BIGINT", "DEFAULT 0 NOT NULL"), + MetaField("row_count", "BIGINT", "DEFAULT 0 NOT NULL"), MetaField("updated_time", "BIGINT", "NOT NULL"), + MetaField("created_on", "BIGINT", "NOT NULL"), MetaField("date", "INT", "DEFAULT -1 NOT NULL"), + }); -} // namespace +} // namespace //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options, const int &mode) - : options_(options), - mode_(mode) { +MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions& options, const int& mode) : options_(options), mode_(mode) { Initialize(); } @@ -173,7 +170,7 @@ MySQLMetaImpl::~MySQLMetaImpl() { } Status -MySQLMetaImpl::NextTableId(std::string &table_id) { +MySQLMetaImpl::NextTableId(std::string& table_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -182,7 +179,7 @@ MySQLMetaImpl::NextTableId(std::string &table_id) { } Status -MySQLMetaImpl::NextFileId(std::string &file_id) { +MySQLMetaImpl::NextFileId(std::string& file_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -201,7 +198,7 @@ MySQLMetaImpl::ValidateMetaSchema() { return; } - auto validate_func = [&](const MetaSchema &schema) { + auto validate_func = [&](const MetaSchema& schema) { mysqlpp::Query query_statement = connectionPtr->query(); query_statement << "DESC " << schema.name() << ";"; @@ -210,14 +207,14 @@ MySQLMetaImpl::ValidateMetaSchema() { try { mysqlpp::StoreQueryResult res = query_statement.store(); for (size_t i = 0; i < res.num_rows(); i++) { - const mysqlpp::Row &row = res[i]; + const mysqlpp::Row& row = res[i]; std::string name, type; row["Field"].to_string(name); row["Type"].to_string(type); exist_fields.push_back(MetaField(name, type, "")); } - } catch (std::exception &e) { + } catch (std::exception& e) { ENGINE_LOG_DEBUG << "Meta table '" << schema.name() << "' not exist and will be created"; } @@ -228,12 +225,12 @@ MySQLMetaImpl::ValidateMetaSchema() { return schema.IsEqual(exist_fields); }; - //verify Tables + // verify Tables if (!validate_func(TABLES_SCHEMA)) { throw Exception(DB_INCOMPATIB_META, "Meta Tables schema is created by Milvus old version"); } - //verufy TableFiles + // verufy TableFiles if (!validate_func(TABLEFILES_SCHEMA)) { throw Exception(DB_INCOMPATIB_META, "Meta TableFiles schema is created by Milvus old version"); } @@ -241,7 +238,7 @@ MySQLMetaImpl::ValidateMetaSchema() { Status MySQLMetaImpl::Initialize() { - //step 1: create db root path + // step 1: create db root path if (!boost::filesystem::is_directory(options_.path_)) { auto ret = boost::filesystem::create_directory(options_.path_); if (!ret) { @@ -253,7 +250,7 @@ MySQLMetaImpl::Initialize() { std::string uri = options_.backend_uri_; - //step 2: parse and check meta uri + // step 2: parse and check meta uri utils::MetaUriInfo uri_info; auto status = utils::ParseMetaUri(uri, uri_info); if (!status.ok()) { @@ -268,7 +265,7 @@ MySQLMetaImpl::Initialize() { throw Exception(DB_INVALID_META_URI, msg); } - //step 3: connect mysql + // step 3: connect mysql int thread_hint = std::thread::hardware_concurrency(); int max_pool_size = (thread_hint == 0) ? 8 : thread_hint; unsigned int port = 0; @@ -276,15 +273,14 @@ MySQLMetaImpl::Initialize() { port = std::stoi(uri_info.port_); } - mysql_connection_pool_ = - std::make_shared(uri_info.db_name_, uri_info.username_, - uri_info.password_, uri_info.host_, port, max_pool_size); + mysql_connection_pool_ = std::make_shared( + uri_info.db_name_, uri_info.username_, uri_info.password_, uri_info.host_, port, max_pool_size); ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(max_pool_size); - //step 4: validate to avoid open old version schema + // step 4: validate to avoid open old version schema ValidateMetaSchema(); - //step 5: create meta tables + // step 5: create meta tables try { if (mode_ != DBOptions::MODE::CLUSTER_READONLY) { CleanUp(); @@ -303,8 +299,8 @@ MySQLMetaImpl::Initialize() { } mysqlpp::Query InitializeQuery = connectionPtr->query(); - InitializeQuery << "CREATE TABLE IF NOT EXISTS " << - TABLES_SCHEMA.name() << " (" << TABLES_SCHEMA.ToString() + ");"; + InitializeQuery << "CREATE TABLE IF NOT EXISTS " << TABLES_SCHEMA.name() << " (" + << TABLES_SCHEMA.ToString() + ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); @@ -312,16 +308,16 @@ MySQLMetaImpl::Initialize() { return HandleException("Initialization Error", InitializeQuery.error()); } - InitializeQuery << "CREATE TABLE IF NOT EXISTS " << - TABLEFILES_SCHEMA.name() << " (" << TABLEFILES_SCHEMA.ToString() + ");"; + InitializeQuery << "CREATE TABLE IF NOT EXISTS " << TABLEFILES_SCHEMA.name() << " (" + << TABLEFILES_SCHEMA.ToString() + ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); if (!InitializeQuery.exec()) { return HandleException("Initialization Error", InitializeQuery.error()); } - } //Scoped Connection - } catch (std::exception &e) { + } // Scoped Connection + } catch (std::exception& e) { return HandleException("GENERAL ERROR DURING INITIALIZATION", e.what()); } @@ -330,8 +326,7 @@ MySQLMetaImpl::Initialize() { // PXU TODO: Temp solution. Will fix later Status -MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, - const DatesT &dates) { +MySQLMetaImpl::DropPartitionsByDates(const std::string& table_id, const DatesT& dates) { if (dates.empty()) { return Status::OK(); } @@ -345,11 +340,11 @@ MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, try { std::stringstream dateListSS; - for (auto &date : dates) { + for (auto& date : dates) { dateListSS << std::to_string(date) << ", "; } std::string dateListStr = dateListSS.str(); - dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", " + dateListStr = dateListStr.substr(0, dateListStr.size() - 2); // remove the last ", " { mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); @@ -360,12 +355,11 @@ MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, mysqlpp::Query dropPartitionsByDatesQuery = connectionPtr->query(); - dropPartitionsByDatesQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << - "updated_time = " << utils::GetMicroSecTimeStamp() << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "date in (" << dateListStr << ");"; + dropPartitionsByDatesQuery << "UPDATE " << META_TABLEFILES << " " + << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," + << "updated_time = " << utils::GetMicroSecTimeStamp() << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "date in (" << dateListStr << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); @@ -373,17 +367,17 @@ MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, return HandleException("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", dropPartitionsByDatesQuery.error()); } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Successfully drop partitions, table id = " << table_schema.table_id_; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", e.what()); } return Status::OK(); } Status -MySQLMetaImpl::CreateTable(TableSchema &table_schema) { +MySQLMetaImpl::CreateTable(TableSchema& table_schema) { try { server::MetricCollector metric; { @@ -398,9 +392,8 @@ MySQLMetaImpl::CreateTable(TableSchema &table_schema) { if (table_schema.table_id_.empty()) { NextTableId(table_schema.table_id_); } else { - createTableQuery << "SELECT state FROM " << - META_TABLES << " " << - "WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ << ";"; + createTableQuery << "SELECT state FROM " << META_TABLES << " " + << "WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); @@ -419,7 +412,7 @@ MySQLMetaImpl::CreateTable(TableSchema &table_schema) { table_schema.id_ = -1; table_schema.created_on_ = utils::GetMicroSecTimeStamp(); - std::string id = "NULL"; //auto-increment + std::string id = "NULL"; // auto-increment std::string table_id = table_schema.table_id_; std::string state = std::to_string(table_schema.state_); std::string dimension = std::to_string(table_schema.dimension_); @@ -430,35 +423,32 @@ MySQLMetaImpl::CreateTable(TableSchema &table_schema) { std::string nlist = std::to_string(table_schema.nlist_); std::string metric_type = std::to_string(table_schema.metric_type_); - createTableQuery << "INSERT INTO " << - META_TABLES << " " << - "VALUES(" << id << ", " << mysqlpp::quote << table_id << ", " << - state << ", " << dimension << ", " << created_on << ", " << - flag << ", " << index_file_size << ", " << engine_type << ", " << - nlist << ", " << metric_type << ");"; + createTableQuery << "INSERT INTO " << META_TABLES << " " + << "VALUES(" << id << ", " << mysqlpp::quote << table_id << ", " << state << ", " + << dimension << ", " << created_on << ", " << flag << ", " << index_file_size << ", " + << engine_type << ", " << nlist << ", " << metric_type << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); if (mysqlpp::SimpleResult res = createTableQuery.execute()) { - table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? + table_schema.id_ = res.insert_id(); // Might need to use SELECT LAST_INSERT_ID()? - //Consume all results to avoid "Commands out of sync" error + // Consume all results to avoid "Commands out of sync" error } else { return HandleException("Add Table Error", createTableQuery.error()); } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Successfully create table: " << table_schema.table_id_; return utils::CreateTablePath(options_, table_schema.table_id_); - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN CREATING TABLE", e.what()); } } Status -MySQLMetaImpl::FilesByType(const std::string &table_id, - const std::vector &file_types, - std::vector &file_ids) { +MySQLMetaImpl::FilesByType(const std::string& table_id, const std::vector& file_types, + std::vector& file_ids) { if (file_types.empty()) { return Status(DB_ERROR, "file types array is empty"); } @@ -483,42 +473,49 @@ MySQLMetaImpl::FilesByType(const std::string &table_id, } mysqlpp::Query hasNonIndexFilesQuery = connectionPtr->query(); - //since table_id is a unique column we just need to check whether it exists or not - hasNonIndexFilesQuery << "SELECT file_id, file_type FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "file_type in (" << types << ");"; + // since table_id is a unique column we just need to check whether it exists or not + hasNonIndexFilesQuery << "SELECT file_id, file_type FROM " << META_TABLEFILES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "file_type in (" << types << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesByType: " << hasNonIndexFilesQuery.str(); res = hasNonIndexFilesQuery.store(); - } //Scoped Connection + } // Scoped Connection if (res.num_rows() > 0) { int raw_count = 0, new_count = 0, new_merge_count = 0, new_index_count = 0; int to_index_count = 0, index_count = 0, backup_count = 0; - for (auto &resRow : res) { + for (auto& resRow : res) { std::string file_id; resRow["file_id"].to_string(file_id); file_ids.push_back(file_id); int32_t file_type = resRow["file_type"]; switch (file_type) { - case (int) TableFileSchema::RAW:raw_count++; + case (int)TableFileSchema::RAW: + raw_count++; break; - case (int) TableFileSchema::NEW:new_count++; + case (int)TableFileSchema::NEW: + new_count++; break; - case (int) TableFileSchema::NEW_MERGE:new_merge_count++; + case (int)TableFileSchema::NEW_MERGE: + new_merge_count++; break; - case (int) TableFileSchema::NEW_INDEX:new_index_count++; + case (int)TableFileSchema::NEW_INDEX: + new_index_count++; break; - case (int) TableFileSchema::TO_INDEX:to_index_count++; + case (int)TableFileSchema::TO_INDEX: + to_index_count++; break; - case (int) TableFileSchema::INDEX:index_count++; + case (int)TableFileSchema::INDEX: + index_count++; break; - case (int) TableFileSchema::BACKUP:backup_count++; + case (int)TableFileSchema::BACKUP: + backup_count++; + break; + default: break; - default:break; } } @@ -527,7 +524,7 @@ MySQLMetaImpl::FilesByType(const std::string &table_id, << " new_index files:" << new_index_count << " to_index files:" << to_index_count << " index files:" << index_count << " backup files:" << backup_count; } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN GET FILE BY TYPE", e.what()); } @@ -535,7 +532,7 @@ MySQLMetaImpl::FilesByType(const std::string &table_id, } Status -MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex &index) { +MySQLMetaImpl::UpdateTableIndex(const std::string& table_id, const TableIndex& index) { try { server::MetricCollector metric; @@ -547,33 +544,31 @@ MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex &i } mysqlpp::Query updateTableIndexParamQuery = connectionPtr->query(); - updateTableIndexParamQuery << "SELECT id, state, dimension, created_on FROM " << - META_TABLES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + updateTableIndexParamQuery << "SELECT id, state, dimension, created_on FROM " << META_TABLES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndex: " << updateTableIndexParamQuery.str(); mysqlpp::StoreQueryResult res = updateTableIndexParamQuery.store(); if (res.num_rows() == 1) { - const mysqlpp::Row &resRow = res[0]; + const mysqlpp::Row& resRow = res[0]; size_t id = resRow["id"]; int32_t state = resRow["state"]; uint16_t dimension = resRow["dimension"]; int64_t created_on = resRow["created_on"]; - updateTableIndexParamQuery << "UPDATE " << - META_TABLES << " " << - "SET id = " << id << ", " << - "state = " << state << ", " << - "dimension = " << dimension << ", " << - "created_on = " << created_on << ", " << - "engine_type = " << index.engine_type_ << ", " << - "nlist = " << index.nlist_ << ", " << - "metric_type = " << index.metric_type_ << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << ";"; + updateTableIndexParamQuery << "UPDATE " << META_TABLES << " " + << "SET id = " << id << ", " + << "state = " << state << ", " + << "dimension = " << dimension << ", " + << "created_on = " << created_on << ", " + << "engine_type = " << index.engine_type_ << ", " + << "nlist = " << index.nlist_ << ", " + << "metric_type = " << index.metric_type_ << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndex: " << updateTableIndexParamQuery.str(); @@ -584,10 +579,10 @@ MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex &i } else { return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Successfully update table index, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE INDEX PARAM", e.what()); } @@ -595,7 +590,7 @@ MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex &i } Status -MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { +MySQLMetaImpl::UpdateTableFlag(const std::string& table_id, int64_t flag) { try { server::MetricCollector metric; @@ -607,20 +602,19 @@ MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { } mysqlpp::Query updateTableFlagQuery = connectionPtr->query(); - updateTableFlagQuery << "UPDATE " << - META_TABLES << " " << - "SET flag = " << flag << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << ";"; + updateTableFlagQuery << "UPDATE " << META_TABLES << " " + << "SET flag = " << flag << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFlag: " << updateTableFlagQuery.str(); if (!updateTableFlagQuery.exec()) { return HandleException("QUERY ERROR WHEN UPDATING TABLE FLAG", updateTableFlagQuery.error()); } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Successfully update table flag, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); } @@ -628,7 +622,7 @@ MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { } Status -MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex &index) { +MySQLMetaImpl::DescribeTableIndex(const std::string& table_id, TableIndex& index) { try { server::MetricCollector metric; @@ -640,17 +634,17 @@ MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex &index } mysqlpp::Query describeTableIndexQuery = connectionPtr->query(); - describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type FROM " << - META_TABLES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type FROM " << META_TABLES + << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTableIndex: " << describeTableIndexQuery.str(); mysqlpp::StoreQueryResult res = describeTableIndexQuery.store(); if (res.num_rows() == 1) { - const mysqlpp::Row &resRow = res[0]; + const mysqlpp::Row& resRow = res[0]; index.engine_type_ = resRow["engine_type"]; index.nlist_ = resRow["nlist"]; @@ -658,8 +652,8 @@ MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex &index } else { return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); } - } //Scoped Connection - } catch (std::exception &e) { + } // Scoped Connection + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); } @@ -667,7 +661,7 @@ MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex &index } Status -MySQLMetaImpl::DropTableIndex(const std::string &table_id) { +MySQLMetaImpl::DropTableIndex(const std::string& table_id) { try { server::MetricCollector metric; @@ -680,13 +674,12 @@ MySQLMetaImpl::DropTableIndex(const std::string &table_id) { mysqlpp::Query dropTableIndexQuery = connectionPtr->query(); - //soft delete index files - dropTableIndexQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << - "updated_time = " << utils::GetMicroSecTimeStamp() << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::INDEX) << ";"; + // soft delete index files + dropTableIndexQuery << "UPDATE " << META_TABLEFILES << " " + << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," + << "updated_time = " << utils::GetMicroSecTimeStamp() << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "file_type = " << std::to_string(TableFileSchema::INDEX) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); @@ -694,13 +687,12 @@ MySQLMetaImpl::DropTableIndex(const std::string &table_id) { return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); } - //set all backup file to raw - dropTableIndexQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET file_type = " << std::to_string(TableFileSchema::RAW) << "," << - "updated_time = " << utils::GetMicroSecTimeStamp() << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::BACKUP) << ";"; + // set all backup file to raw + dropTableIndexQuery << "UPDATE " << META_TABLEFILES << " " + << "SET file_type = " << std::to_string(TableFileSchema::RAW) << "," + << "updated_time = " << utils::GetMicroSecTimeStamp() << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "file_type = " << std::to_string(TableFileSchema::BACKUP) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); @@ -708,23 +700,22 @@ MySQLMetaImpl::DropTableIndex(const std::string &table_id) { return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); } - //set table index type to raw - dropTableIndexQuery << "UPDATE " << - META_TABLES << " " << - "SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," << - "nlist = " << std::to_string(DEFAULT_NLIST) << ", " << - "metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << ";"; + // set table index type to raw + dropTableIndexQuery << "UPDATE " << META_TABLES << " " + << "SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," + << "nlist = " << std::to_string(DEFAULT_NLIST) << ", " + << "metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); if (!dropTableIndexQuery.exec()) { return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DROPPING TABLE INDEX", e.what()); } @@ -732,7 +723,7 @@ MySQLMetaImpl::DropTableIndex(const std::string &table_id) { } Status -MySQLMetaImpl::DeleteTable(const std::string &table_id) { +MySQLMetaImpl::DeleteTable(const std::string& table_id) { try { server::MetricCollector metric; { @@ -742,27 +733,26 @@ MySQLMetaImpl::DeleteTable(const std::string &table_id) { return Status(DB_ERROR, "Failed to connect to database server"); } - //soft delete table + // soft delete table mysqlpp::Query deleteTableQuery = connectionPtr->query(); -// - deleteTableQuery << "UPDATE " << - META_TABLES << " " << - "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << ";"; + // + deleteTableQuery << "UPDATE " << META_TABLES << " " + << "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); if (!deleteTableQuery.exec()) { return HandleException("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); } - } //Scoped Connection + } // Scoped Connection if (mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { DeleteTableFiles(table_id); } ENGINE_LOG_DEBUG << "Successfully delete table, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DELETING TABLE", e.what()); } @@ -770,7 +760,7 @@ MySQLMetaImpl::DeleteTable(const std::string &table_id) { } Status -MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { +MySQLMetaImpl::DeleteTableFiles(const std::string& table_id) { try { server::MetricCollector metric; { @@ -780,25 +770,24 @@ MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { return Status(DB_ERROR, "Failed to connect to database server"); } - //soft delete table files + // soft delete table files mysqlpp::Query deleteTableFilesQuery = connectionPtr->query(); // - deleteTableFilesQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << - "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + deleteTableFilesQuery << "UPDATE " << META_TABLEFILES << " " + << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " + << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); if (!deleteTableFilesQuery.exec()) { return HandleException("QUERY ERROR WHEN DELETING TABLE FILES", deleteTableFilesQuery.error()); } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Successfully delete table files, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DELETING TABLE FILES", e.what()); } @@ -806,7 +795,7 @@ MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { } Status -MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { +MySQLMetaImpl::DescribeTable(TableSchema& table_schema) { try { server::MetricCollector metric; mysqlpp::StoreQueryResult res; @@ -820,19 +809,19 @@ MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { mysqlpp::Query describeTableQuery = connectionPtr->query(); describeTableQuery << "SELECT id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type " - << " FROM " << META_TABLES << " " << - "WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + << " FROM " << META_TABLES << " " + << "WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ << " " + << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); res = describeTableQuery.store(); - } //Scoped Connection + } // Scoped Connection if (res.num_rows() == 1) { - const mysqlpp::Row &resRow = res[0]; + const mysqlpp::Row& resRow = res[0]; - table_schema.id_ = resRow["id"]; //implicit conversion + table_schema.id_ = resRow["id"]; // implicit conversion table_schema.state_ = resRow["state"]; @@ -852,7 +841,7 @@ MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { } else { return Status(DB_NOT_FOUND, "Table " + table_schema.table_id_ + " not found"); } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DESCRIBING TABLE", e.what()); } @@ -860,7 +849,7 @@ MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { } Status -MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { +MySQLMetaImpl::HasTable(const std::string& table_id, bool& has_or_not) { try { server::MetricCollector metric; mysqlpp::StoreQueryResult res; @@ -872,22 +861,22 @@ MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { } mysqlpp::Query hasTableQuery = connectionPtr->query(); - //since table_id is a unique column we just need to check whether it exists or not - hasTableQuery << "SELECT EXISTS " << - "(SELECT 1 FROM " << - META_TABLES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << - "AS " << mysqlpp::quote << "check" << ";"; + // since table_id is a unique column we just need to check whether it exists or not + hasTableQuery << "SELECT EXISTS " + << "(SELECT 1 FROM " << META_TABLES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " " + << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " + << "AS " << mysqlpp::quote << "check" + << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); res = hasTableQuery.store(); - } //Scoped Connection + } // Scoped Connection int check = res[0]["check"]; has_or_not = (check == 1); - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", e.what()); } @@ -895,7 +884,7 @@ MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { } Status -MySQLMetaImpl::AllTables(std::vector &table_schema_array) { +MySQLMetaImpl::AllTables(std::vector& table_schema_array) { try { server::MetricCollector metric; mysqlpp::StoreQueryResult res; @@ -908,19 +897,18 @@ MySQLMetaImpl::AllTables(std::vector &table_schema_array) { mysqlpp::Query allTablesQuery = connectionPtr->query(); allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type FROM " - << - META_TABLES << " " << - "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + << META_TABLES << " " + << "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); res = allTablesQuery.store(); - } //Scoped Connection + } // Scoped Connection - for (auto &resRow : res) { + for (auto& resRow : res) { TableSchema table_schema; - table_schema.id_ = resRow["id"]; //implicit conversion + table_schema.id_ = resRow["id"]; // implicit conversion std::string table_id; resRow["table_id"].to_string(table_id); @@ -938,7 +926,7 @@ MySQLMetaImpl::AllTables(std::vector &table_schema_array) { table_schema_array.emplace_back(table_schema); } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DESCRIBING ALL TABLES", e.what()); } @@ -946,7 +934,7 @@ MySQLMetaImpl::AllTables(std::vector &table_schema_array) { } Status -MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { +MySQLMetaImpl::CreateTableFile(TableFileSchema& file_schema) { if (file_schema.date_ == EmptyDate) { file_schema.date_ = utils::GetDate(); } @@ -971,7 +959,7 @@ MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { file_schema.nlist_ = table_schema.nlist_; file_schema.metric_type_ = table_schema.metric_type_; - std::string id = "NULL"; //auto-increment + std::string id = "NULL"; // auto-increment std::string table_id = file_schema.table_id_; std::string engine_type = std::to_string(file_schema.engine_type_); std::string file_id = file_schema.file_id_; @@ -991,33 +979,31 @@ MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { mysqlpp::Query createTableFileQuery = connectionPtr->query(); - createTableFileQuery << "INSERT INTO " << - META_TABLEFILES << " " << - "VALUES(" << id << ", " << mysqlpp::quote << table_id << - ", " << engine_type << ", " << - mysqlpp::quote << file_id << ", " << file_type << ", " << file_size << ", " << - row_count << ", " << updated_time << ", " << created_on << ", " << date << ");"; + createTableFileQuery << "INSERT INTO " << META_TABLEFILES << " " + << "VALUES(" << id << ", " << mysqlpp::quote << table_id << ", " << engine_type << ", " + << mysqlpp::quote << file_id << ", " << file_type << ", " << file_size << ", " + << row_count << ", " << updated_time << ", " << created_on << ", " << date << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); if (mysqlpp::SimpleResult res = createTableFileQuery.execute()) { - file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? + file_schema.id_ = res.insert_id(); // Might need to use SELECT LAST_INSERT_ID()? - //Consume all results to avoid "Commands out of sync" error + // Consume all results to avoid "Commands out of sync" error } else { return HandleException("QUERY ERROR WHEN CREATING TABLE FILE", createTableFileQuery.error()); } - } // Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Successfully create table file, file id = " << file_schema.file_id_; return utils::CreateTableFilePath(options_, file_schema); - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN CREATING TABLE FILE", e.what()); } } Status -MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { +MySQLMetaImpl::FilesToIndex(TableFilesSchema& files) { files.clear(); try { @@ -1033,20 +1019,19 @@ MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { mysqlpp::Query filesToIndexQuery = connectionPtr->query(); filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " - << - META_TABLEFILES << " " << - "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; + << META_TABLEFILES << " " + << "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); res = filesToIndexQuery.store(); - } //Scoped Connection + } // Scoped Connection Status ret; std::map groups; TableFileSchema table_file; - for (auto &resRow : res) { - table_file.id_ = resRow["id"]; //implicit conversion + for (auto& resRow : res) { + table_file.id_ = resRow["id"]; // implicit conversion std::string table_id; resRow["table_id"].to_string(table_id); @@ -1095,16 +1080,14 @@ MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-index files"; } return ret; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", e.what()); } } Status -MySQLMetaImpl::FilesToSearch(const std::string &table_id, - const std::vector &ids, - const DatesT &partition, - DatePartionedTableFilesSchema &files) { +MySQLMetaImpl::FilesToSearch(const std::string& table_id, const std::vector& ids, const DatesT& partition, + DatePartionedTableFilesSchema& files) { files.clear(); try { @@ -1119,41 +1102,43 @@ MySQLMetaImpl::FilesToSearch(const std::string &table_id, mysqlpp::Query filesToSearchQuery = connectionPtr->query(); filesToSearchQuery - << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id; + << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date FROM " + << META_TABLEFILES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id; if (!partition.empty()) { std::stringstream partitionListSS; - for (auto &date : partition) { + for (auto& date : partition) { partitionListSS << std::to_string(date) << ", "; } std::string partitionListStr = partitionListSS.str(); - partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); //remove the last ", " - filesToSearchQuery << " AND " << "date IN (" << partitionListStr << ")"; + partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); // remove the last ", " + filesToSearchQuery << " AND " + << "date IN (" << partitionListStr << ")"; } if (!ids.empty()) { std::stringstream idSS; - for (auto &id : ids) { + for (auto& id : ids) { idSS << "id = " << std::to_string(id) << " OR "; } std::string idStr = idSS.str(); - idStr = idStr.substr(0, idStr.size() - 4); //remove the last " OR " + idStr = idStr.substr(0, idStr.size() - 4); // remove the last " OR " - filesToSearchQuery << " AND " << "(" << idStr << ")"; + filesToSearchQuery << " AND " + << "(" << idStr << ")"; } // End - filesToSearchQuery << " AND " << - "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << - "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + filesToSearchQuery << " AND " + << "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " + << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " + << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); res = filesToSearchQuery.store(); - } //Scoped Connection + } // Scoped Connection TableSchema table_schema; table_schema.table_id_ = table_id; @@ -1164,8 +1149,8 @@ MySQLMetaImpl::FilesToSearch(const std::string &table_id, Status ret; TableFileSchema table_file; - for (auto &resRow : res) { - table_file.id_ = resRow["id"]; //implicit conversion + for (auto& resRow : res) { + table_file.id_ = resRow["id"]; // implicit conversion std::string table_id_str; resRow["table_id"].to_string(table_id_str); @@ -1210,20 +1195,19 @@ MySQLMetaImpl::FilesToSearch(const std::string &table_id, ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-search files"; } return ret; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", e.what()); } } Status -MySQLMetaImpl::FilesToMerge(const std::string &table_id, - DatePartionedTableFilesSchema &files) { +MySQLMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) { files.clear(); try { server::MetricCollector metric; - //check table existence + // check table existence TableSchema table_schema; table_schema.table_id_ = table_id; auto status = DescribeTable(table_schema); @@ -1242,26 +1226,26 @@ MySQLMetaImpl::FilesToMerge(const std::string &table_id, mysqlpp::Query filesToMergeQuery = connectionPtr->query(); filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on FROM " - << - META_TABLEFILES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::RAW) << " " << - "ORDER BY row_count DESC" << ";"; + << META_TABLEFILES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "file_type = " << std::to_string(TableFileSchema::RAW) << " " + << "ORDER BY row_count DESC" + << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); res = filesToMergeQuery.store(); - } //Scoped Connection + } // Scoped Connection Status ret; - for (auto &resRow : res) { + for (auto& resRow : res) { TableFileSchema table_file; table_file.file_size_ = resRow["file_size"]; if (table_file.file_size_ >= table_schema.index_file_size_) { - continue;//skip large file + continue; // skip large file } - table_file.id_ = resRow["id"]; //implicit conversion + table_file.id_ = resRow["id"]; // implicit conversion std::string table_id_str; resRow["table_id"].to_string(table_id_str); @@ -1306,25 +1290,24 @@ MySQLMetaImpl::FilesToMerge(const std::string &table_id, ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-merge files"; } return ret; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", e.what()); } } Status -MySQLMetaImpl::GetTableFiles(const std::string &table_id, - const std::vector &ids, - TableFilesSchema &table_files) { +MySQLMetaImpl::GetTableFiles(const std::string& table_id, const std::vector& ids, + TableFilesSchema& table_files) { if (ids.empty()) { return Status::OK(); } std::stringstream idSS; - for (auto &id : ids) { + for (auto& id : ids) { idSS << "id = " << std::to_string(id) << " OR "; } std::string idStr = idSS.str(); - idStr = idStr.substr(0, idStr.size() - 4); //remove the last " OR " + idStr = idStr.substr(0, idStr.size() - 4); // remove the last " OR " try { mysqlpp::StoreQueryResult res; @@ -1337,23 +1320,23 @@ MySQLMetaImpl::GetTableFiles(const std::string &table_id, mysqlpp::Query getTableFileQuery = connectionPtr->query(); getTableFileQuery - << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "(" << idStr << ") AND " << - "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " + << META_TABLEFILES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "(" << idStr << ") AND " + << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); res = getTableFileQuery.store(); - } //Scoped Connection + } // Scoped Connection TableSchema table_schema; table_schema.table_id_ = table_id; DescribeTable(table_schema); Status ret; - for (auto &resRow : res) { + for (auto& resRow : res) { TableFileSchema file_schema; file_schema.id_ = resRow["id"]; @@ -1391,7 +1374,7 @@ MySQLMetaImpl::GetTableFiles(const std::string &table_id, ENGINE_LOG_DEBUG << "Get table files by id"; return ret; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN RETRIEVING TABLE FILES", e.what()); } } @@ -1399,14 +1382,14 @@ MySQLMetaImpl::GetTableFiles(const std::string &table_id, // PXU TODO: Support Swap Status MySQLMetaImpl::Archive() { - auto &criterias = options_.archive_conf_.GetCriterias(); + auto& criterias = options_.archive_conf_.GetCriterias(); if (criterias.empty()) { return Status::OK(); } - for (auto &kv : criterias) { - auto &criteria = kv.first; - auto &limit = kv.second; + for (auto& kv : criterias) { + auto& criteria = kv.first; + auto& limit = kv.second; if (criteria == engine::ARCHIVE_CONF_DAYS) { size_t usecs = limit * D_SEC * US_PS; int64_t now = utils::GetMicroSecTimeStamp(); @@ -1419,11 +1402,10 @@ MySQLMetaImpl::Archive() { } mysqlpp::Query archiveQuery = connectionPtr->query(); - archiveQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << - "WHERE created_on < " << std::to_string(now - usecs) << " AND " << - "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + archiveQuery << "UPDATE " << META_TABLEFILES << " " + << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " + << "WHERE created_on < " << std::to_string(now - usecs) << " AND " + << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); @@ -1432,7 +1414,7 @@ MySQLMetaImpl::Archive() { } ENGINE_LOG_DEBUG << "Archive old files"; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DURING ARCHIVE", e.what()); } } @@ -1451,7 +1433,7 @@ MySQLMetaImpl::Archive() { } Status -MySQLMetaImpl::Size(uint64_t &result) { +MySQLMetaImpl::Size(uint64_t& result) { result = 0; try { @@ -1464,21 +1446,20 @@ MySQLMetaImpl::Size(uint64_t &result) { } mysqlpp::Query getSizeQuery = connectionPtr->query(); - getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum FROM " << - META_TABLEFILES << " " << - "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum FROM " << META_TABLEFILES << " " + << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); res = getSizeQuery.store(); - } //Scoped Connection + } // Scoped Connection if (res.empty()) { result = 0; } else { result = res[0]["sum"]; } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN RETRIEVING SIZE", e.what()); } @@ -1503,11 +1484,10 @@ MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { } mysqlpp::Query discardFilesQuery = connectionPtr->query(); - discardFilesQuery << "SELECT id, file_size FROM " << - META_TABLEFILES << " " << - "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << - "ORDER BY id ASC " << - "LIMIT 10;"; + discardFilesQuery << "SELECT id, file_size FROM " << META_TABLEFILES << " " + << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " + << "ORDER BY id ASC " + << "LIMIT 10;"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); @@ -1518,7 +1498,7 @@ MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { TableFileSchema table_file; std::stringstream idsToDiscardSS; - for (auto &resRow : res) { + for (auto& resRow : res) { if (to_discard_size <= 0) { break; } @@ -1531,13 +1511,12 @@ MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { } std::string idsToDiscardStr = idsToDiscardSS.str(); - idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR " + idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); // remove the last " OR " - discardFilesQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << - "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << - "WHERE " << idsToDiscardStr << ";"; + discardFilesQuery << "UPDATE " << META_TABLEFILES << " " + << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " + << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " + << "WHERE " << idsToDiscardStr << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); @@ -1545,17 +1524,17 @@ MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { if (!status) { return HandleException("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error()); } - } //Scoped Connection + } // Scoped Connection return DiscardFiles(to_discard_size); - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DISCARDING FILES", e.what()); } } -//ZR: this function assumes all fields in file_schema have value +// ZR: this function assumes all fields in file_schema have value Status -MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { +MySQLMetaImpl::UpdateTableFile(TableFileSchema& file_schema) { file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); try { @@ -1569,11 +1548,10 @@ MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { mysqlpp::Query updateTableFileQuery = connectionPtr->query(); - //if the table has been deleted, just mark the table file as TO_DELETE - //clean thread will delete the file later - updateTableFileQuery << "SELECT state FROM " << - META_TABLES << " " << - "WHERE table_id = " << mysqlpp::quote << file_schema.table_id_ << ";"; + // if the table has been deleted, just mark the table file as TO_DELETE + // clean thread will delete the file later + updateTableFileQuery << "SELECT state FROM " << META_TABLES << " " + << "WHERE table_id = " << mysqlpp::quote << file_schema.table_id_ << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); @@ -1599,18 +1577,17 @@ MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); - updateTableFileQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET table_id = " << mysqlpp::quote << table_id << ", " << - "engine_type = " << engine_type << ", " << - "file_id = " << mysqlpp::quote << file_id << ", " << - "file_type = " << file_type << ", " << - "file_size = " << file_size << ", " << - "row_count = " << row_count << ", " << - "updated_time = " << updated_time << ", " << - "created_on = " << created_on << ", " << - "date = " << date << " " << - "WHERE id = " << id << ";"; + updateTableFileQuery << "UPDATE " << META_TABLEFILES << " " + << "SET table_id = " << mysqlpp::quote << table_id << ", " + << "engine_type = " << engine_type << ", " + << "file_id = " << mysqlpp::quote << file_id << ", " + << "file_type = " << file_type << ", " + << "file_size = " << file_size << ", " + << "row_count = " << row_count << ", " + << "updated_time = " << updated_time << ", " + << "created_on = " << created_on << ", " + << "date = " << date << " " + << "WHERE id = " << id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); @@ -1618,10 +1595,10 @@ MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; return HandleException("QUERY ERROR WHEN UPDATING TABLE FILE", updateTableFileQuery.error()); } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Update single table file, file id = " << file_schema.file_id_; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILE", e.what()); } @@ -1629,7 +1606,7 @@ MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { } Status -MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { +MySQLMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { try { mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); @@ -1639,11 +1616,10 @@ MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { mysqlpp::Query updateTableFilesToIndexQuery = connectionPtr->query(); - updateTableFilesToIndexQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; + updateTableFilesToIndexQuery << "UPDATE " << META_TABLEFILES << " " + << "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFilesToIndex: " << updateTableFilesToIndexQuery.str(); @@ -1653,7 +1629,7 @@ MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { } ENGINE_LOG_DEBUG << "Update files to to_index, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX", e.what()); } @@ -1661,7 +1637,7 @@ MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { } Status -MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { +MySQLMetaImpl::UpdateTableFiles(TableFilesSchema& files) { try { server::MetricCollector metric; { @@ -1674,17 +1650,17 @@ MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { mysqlpp::Query updateTableFilesQuery = connectionPtr->query(); std::map has_tables; - for (auto &file_schema : files) { + for (auto& file_schema : files) { if (has_tables.find(file_schema.table_id_) != has_tables.end()) { continue; } - updateTableFilesQuery << "SELECT EXISTS " << - "(SELECT 1 FROM " << - META_TABLES << " " << - "WHERE table_id = " << mysqlpp::quote << file_schema.table_id_ << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << - "AS " << mysqlpp::quote << "check" << ";"; + updateTableFilesQuery << "SELECT EXISTS " + << "(SELECT 1 FROM " << META_TABLES << " " + << "WHERE table_id = " << mysqlpp::quote << file_schema.table_id_ << " " + << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " + << "AS " << mysqlpp::quote << "check" + << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); @@ -1694,7 +1670,7 @@ MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { has_tables[file_schema.table_id_] = (check == 1); } - for (auto &file_schema : files) { + for (auto& file_schema : files) { if (!has_tables[file_schema.table_id_]) { file_schema.file_type_ = TableFileSchema::TO_DELETE; } @@ -1711,18 +1687,17 @@ MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); - updateTableFilesQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET table_id = " << mysqlpp::quote << table_id << ", " << - "engine_type = " << engine_type << ", " << - "file_id = " << mysqlpp::quote << file_id << ", " << - "file_type = " << file_type << ", " << - "file_size = " << file_size << ", " << - "row_count = " << row_count << ", " << - "updated_time = " << updated_time << ", " << - "created_on = " << created_on << ", " << - "date = " << date << " " << - "WHERE id = " << id << ";"; + updateTableFilesQuery << "UPDATE " << META_TABLEFILES << " " + << "SET table_id = " << mysqlpp::quote << table_id << ", " + << "engine_type = " << engine_type << ", " + << "file_id = " << mysqlpp::quote << file_id << ", " + << "file_type = " << file_type << ", " + << "file_size = " << file_size << ", " + << "row_count = " << row_count << ", " + << "updated_time = " << updated_time << ", " + << "created_on = " << created_on << ", " + << "date = " << date << " " + << "WHERE id = " << id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); @@ -1730,10 +1705,10 @@ MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { return HandleException("QUERY ERROR WHEN UPDATING TABLE FILES", updateTableFilesQuery.error()); } } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Update " << files.size() << " table files"; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES", e.what()); } @@ -1745,7 +1720,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { auto now = utils::GetMicroSecTimeStamp(); std::set table_ids; - //remove to_delete files + // remove to_delete files try { server::MetricCollector metric; @@ -1757,10 +1732,9 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date FROM " << - META_TABLEFILES << " " << - "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << - "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; + cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date FROM " << META_TABLEFILES << " " + << "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " + << "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1769,8 +1743,8 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { TableFileSchema table_file; std::vector idsToDelete; - for (auto &resRow : res) { - table_file.id_ = resRow["id"]; //implicit conversion + for (auto& resRow : res) { + table_file.id_ = resRow["id"]; // implicit conversion std::string table_id; resRow["table_id"].to_string(table_id); @@ -1793,15 +1767,14 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { if (!idsToDelete.empty()) { std::stringstream idsToDeleteSS; - for (auto &id : idsToDelete) { + for (auto& id : idsToDelete) { idsToDeleteSS << "id = " << id << " OR "; } std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM " << - META_TABLEFILES << " " << - "WHERE " << idsToDeleteStr << ";"; + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); // remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM " << META_TABLEFILES << " " + << "WHERE " << idsToDeleteStr << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1814,12 +1787,12 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { if (res.size() > 0) { ENGINE_LOG_DEBUG << "Clean " << res.size() << " files deleted in " << seconds << " seconds"; } - } //Scoped Connection - } catch (std::exception &e) { + } // Scoped Connection + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", e.what()); } - //remove to_delete tables + // remove to_delete tables try { server::MetricCollector metric; @@ -1831,9 +1804,8 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id FROM " << - META_TABLES << " " << - "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; + cleanUpFilesWithTTLQuery << "SELECT id, table_id FROM " << META_TABLES << " " + << "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1841,20 +1813,19 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { if (!res.empty()) { std::stringstream idsToDeleteSS; - for (auto &resRow : res) { + for (auto& resRow : res) { size_t id = resRow["id"]; std::string table_id; resRow["table_id"].to_string(table_id); - utils::DeleteTablePath(options_, table_id, false);//only delete empty folder + utils::DeleteTablePath(options_, table_id, false); // only delete empty folder idsToDeleteSS << "id = " << std::to_string(id) << " OR "; } std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM " << - META_TABLES << " " << - "WHERE " << idsToDeleteStr << ";"; + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); // remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM " << META_TABLES << " " + << "WHERE " << idsToDeleteStr << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1867,13 +1838,13 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { if (res.size() > 0) { ENGINE_LOG_DEBUG << "Remove " << res.size() << " tables from meta"; } - } //Scoped Connection - } catch (std::exception &e) { + } // Scoped Connection + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); } - //remove deleted table folder - //don't remove table folder until all its files has been deleted + // remove deleted table folder + // don't remove table folder until all its files has been deleted try { server::MetricCollector metric; @@ -1884,11 +1855,10 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return Status(DB_ERROR, "Failed to connect to database server"); } - for (auto &table_id : table_ids) { + for (auto& table_id : table_ids) { mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT file_id FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << ";"; + cleanUpFilesWithTTLQuery << "SELECT file_id FROM " << META_TABLEFILES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1903,7 +1873,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { ENGINE_LOG_DEBUG << "Remove " << table_ids.size() << " tables folder"; } } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); } @@ -1920,10 +1890,10 @@ MySQLMetaImpl::CleanUp() { } mysqlpp::Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "SELECT table_name " << - "FROM information_schema.tables " << - "WHERE table_schema = " << mysqlpp::quote << mysql_connection_pool_->getDB() << " " << - "AND table_name = " << mysqlpp::quote << META_TABLEFILES << ";"; + cleanUpQuery << "SELECT table_name " + << "FROM information_schema.tables " + << "WHERE table_schema = " << mysqlpp::quote << mysql_connection_pool_->getDB() << " " + << "AND table_name = " << mysqlpp::quote << META_TABLEFILES << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); @@ -1932,9 +1902,8 @@ MySQLMetaImpl::CleanUp() { if (!res.empty()) { ENGINE_LOG_DEBUG << "Remove table file type as NEW"; cleanUpQuery << "DELETE FROM " << META_TABLEFILES << " WHERE file_type IN (" - << std::to_string(TableFileSchema::NEW) << "," - << std::to_string(TableFileSchema::NEW_MERGE) << "," - << std::to_string(TableFileSchema::NEW_INDEX) << ");"; + << std::to_string(TableFileSchema::NEW) << "," << std::to_string(TableFileSchema::NEW_MERGE) + << "," << std::to_string(TableFileSchema::NEW_INDEX) << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); @@ -1946,7 +1915,7 @@ MySQLMetaImpl::CleanUp() { if (res.size() > 0) { ENGINE_LOG_DEBUG << "Clean " << res.size() << " files"; } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN CLEANING UP FILES", e.what()); } @@ -1954,7 +1923,7 @@ MySQLMetaImpl::CleanUp() { } Status -MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { +MySQLMetaImpl::Count(const std::string& table_id, uint64_t& result) { try { server::MetricCollector metric; @@ -1975,24 +1944,23 @@ MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { } mysqlpp::Query countQuery = connectionPtr->query(); - countQuery << "SELECT row_count FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << - "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + countQuery << "SELECT row_count FROM " << META_TABLEFILES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " + << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " + << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str(); res = countQuery.store(); - } //Scoped Connection + } // Scoped Connection result = 0; - for (auto &resRow : res) { + for (auto& resRow : res) { size_t size = resRow["row_count"]; result += size; } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN RETRIEVING COUNT", e.what()); } @@ -2019,12 +1987,12 @@ MySQLMetaImpl::DropAll() { } else { return HandleException("QUERY ERROR WHEN DROPPING ALL", dropTableQuery.error()); } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DROPPING ALL", e.what()); } } -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index d9d67fd748..48bdf5b9ab 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -18,14 +18,14 @@ #pragma once #include "Meta.h" -#include "db/Options.h" #include "MySQLConnectionPool.h" +#include "db/Options.h" #include -#include -#include -#include #include +#include +#include +#include namespace zilliz { namespace milvus { @@ -34,77 +34,101 @@ namespace meta { class MySQLMetaImpl : public Meta { public: - MySQLMetaImpl(const DBMetaOptions &options, const int &mode); + MySQLMetaImpl(const DBMetaOptions& options, const int& mode); ~MySQLMetaImpl(); - Status CreateTable(TableSchema &table_schema) override; + Status + CreateTable(TableSchema& table_schema) override; - Status DescribeTable(TableSchema &table_schema) override; + Status + DescribeTable(TableSchema& table_schema) override; - Status HasTable(const std::string &table_id, bool &has_or_not) override; + Status + HasTable(const std::string& table_id, bool& has_or_not) override; - Status AllTables(std::vector &table_schema_array) override; + Status + AllTables(std::vector& table_schema_array) override; - Status DeleteTable(const std::string &table_id) override; + Status + DeleteTable(const std::string& table_id) override; - Status DeleteTableFiles(const std::string &table_id) override; + Status + DeleteTableFiles(const std::string& table_id) override; - Status CreateTableFile(TableFileSchema &file_schema) override; + Status + CreateTableFile(TableFileSchema& file_schema) override; - Status DropPartitionsByDates(const std::string &table_id, - const DatesT &dates) override; + Status + DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override; - Status GetTableFiles(const std::string &table_id, - const std::vector &ids, - TableFilesSchema &table_files) override; + Status + GetTableFiles(const std::string& table_id, const std::vector& ids, TableFilesSchema& table_files) override; - Status FilesByType(const std::string &table_id, - const std::vector &file_types, - std::vector &file_ids) override; + Status + FilesByType(const std::string& table_id, const std::vector& file_types, + std::vector& file_ids) override; - Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override; + Status + UpdateTableIndex(const std::string& table_id, const TableIndex& index) override; - Status UpdateTableFlag(const std::string &table_id, int64_t flag) override; + Status + UpdateTableFlag(const std::string& table_id, int64_t flag) override; - Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override; + Status + DescribeTableIndex(const std::string& table_id, TableIndex& index) override; - Status DropTableIndex(const std::string &table_id) override; + Status + DropTableIndex(const std::string& table_id) override; - Status UpdateTableFile(TableFileSchema &file_schema) override; + Status + UpdateTableFile(TableFileSchema& file_schema) override; - Status UpdateTableFilesToIndex(const std::string &table_id) override; + Status + UpdateTableFilesToIndex(const std::string& table_id) override; - Status UpdateTableFiles(TableFilesSchema &files) override; + Status + UpdateTableFiles(TableFilesSchema& files) override; - Status FilesToSearch(const std::string &table_id, - const std::vector &ids, - const DatesT &partition, - DatePartionedTableFilesSchema &files) override; + Status + FilesToSearch(const std::string& table_id, const std::vector& ids, const DatesT& partition, + DatePartionedTableFilesSchema& files) override; - Status FilesToMerge(const std::string &table_id, - DatePartionedTableFilesSchema &files) override; + Status + FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) override; - Status FilesToIndex(TableFilesSchema &) override; + Status + FilesToIndex(TableFilesSchema&) override; - Status Archive() override; + Status + Archive() override; - Status Size(uint64_t &result) override; + Status + Size(uint64_t& result) override; - Status CleanUp() override; + Status + CleanUp() override; - Status CleanUpFilesWithTTL(uint16_t seconds) override; + Status + CleanUpFilesWithTTL(uint16_t seconds) override; - Status DropAll() override; + Status + DropAll() override; - Status Count(const std::string &table_id, uint64_t &result) override; + Status + Count(const std::string& table_id, uint64_t& result) override; private: - Status NextFileId(std::string &file_id); - Status NextTableId(std::string &table_id); - Status DiscardFiles(int64_t to_discard_size); + Status + NextFileId(std::string& file_id); + Status + NextTableId(std::string& table_id); + Status + DiscardFiles(int64_t to_discard_size); - void ValidateMetaSchema(); - Status Initialize(); + void + ValidateMetaSchema(); + Status + Initialize(); private: const DBMetaOptions options_; @@ -113,10 +137,10 @@ class MySQLMetaImpl : public Meta { std::shared_ptr mysql_connection_pool_; bool safe_grab_ = false; -// std::mutex connectionMutex_; -}; // DBMetaImpl + // std::mutex connectionMutex_; +}; // DBMetaImpl -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/SqliteMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h index 978e5e5a4b..70f22db281 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -21,8 +21,8 @@ #include "db/Options.h" #include -#include #include +#include namespace zilliz { namespace milvus { @@ -30,86 +30,112 @@ namespace engine { namespace meta { auto -StoragePrototype(const std::string &path); +StoragePrototype(const std::string& path); class SqliteMetaImpl : public Meta { public: - explicit SqliteMetaImpl(const DBMetaOptions &options); + explicit SqliteMetaImpl(const DBMetaOptions& options); ~SqliteMetaImpl(); - Status CreateTable(TableSchema &table_schema) override; + Status + CreateTable(TableSchema& table_schema) override; - Status DescribeTable(TableSchema &table_schema) override; + Status + DescribeTable(TableSchema& table_schema) override; - Status HasTable(const std::string &table_id, bool &has_or_not) override; + Status + HasTable(const std::string& table_id, bool& has_or_not) override; - Status AllTables(std::vector &table_schema_array) override; + Status + AllTables(std::vector& table_schema_array) override; - Status DeleteTable(const std::string &table_id) override; + Status + DeleteTable(const std::string& table_id) override; - Status DeleteTableFiles(const std::string &table_id) override; + Status + DeleteTableFiles(const std::string& table_id) override; - Status CreateTableFile(TableFileSchema &file_schema) override; + Status + CreateTableFile(TableFileSchema& file_schema) override; - Status DropPartitionsByDates(const std::string &table_id, const DatesT &dates) override; + Status + DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override; - Status GetTableFiles(const std::string &table_id, - const std::vector &ids, - TableFilesSchema &table_files) override; + Status + GetTableFiles(const std::string& table_id, const std::vector& ids, TableFilesSchema& table_files) override; - Status FilesByType(const std::string &table_id, - const std::vector &file_types, - std::vector &file_ids) override; + Status + FilesByType(const std::string& table_id, const std::vector& file_types, + std::vector& file_ids) override; - Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override; + Status + UpdateTableIndex(const std::string& table_id, const TableIndex& index) override; - Status UpdateTableFlag(const std::string &table_id, int64_t flag) override; + Status + UpdateTableFlag(const std::string& table_id, int64_t flag) override; - Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override; + Status + DescribeTableIndex(const std::string& table_id, TableIndex& index) override; - Status DropTableIndex(const std::string &table_id) override; + Status + DropTableIndex(const std::string& table_id) override; - Status UpdateTableFilesToIndex(const std::string &table_id) override; + Status + UpdateTableFilesToIndex(const std::string& table_id) override; - Status UpdateTableFile(TableFileSchema &file_schema) override; + Status + UpdateTableFile(TableFileSchema& file_schema) override; - Status UpdateTableFiles(TableFilesSchema &files) override; + Status + UpdateTableFiles(TableFilesSchema& files) override; - Status FilesToSearch(const std::string &table_id, - const std::vector &ids, - const DatesT &partition, - DatePartionedTableFilesSchema &files) override; + Status + FilesToSearch(const std::string& table_id, const std::vector& ids, const DatesT& partition, + DatePartionedTableFilesSchema& files) override; - Status FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) override; + Status + FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) override; - Status FilesToIndex(TableFilesSchema &) override; + Status + FilesToIndex(TableFilesSchema&) override; - Status Archive() override; + Status + Archive() override; - Status Size(uint64_t &result) override; + Status + Size(uint64_t& result) override; - Status CleanUp() override; + Status + CleanUp() override; - Status CleanUpFilesWithTTL(uint16_t seconds) override; + Status + CleanUpFilesWithTTL(uint16_t seconds) override; - Status DropAll() override; + Status + DropAll() override; - Status Count(const std::string &table_id, uint64_t &result) override; + Status + Count(const std::string& table_id, uint64_t& result) override; private: - Status NextFileId(std::string &file_id); - Status NextTableId(std::string &table_id); - Status DiscardFiles(int64_t to_discard_size); + Status + NextFileId(std::string& file_id); + Status + NextTableId(std::string& table_id); + Status + DiscardFiles(int64_t to_discard_size); - void ValidateMetaSchema(); - Status Initialize(); + void + ValidateMetaSchema(); + Status + Initialize(); private: const DBMetaOptions options_; std::mutex meta_mutex_; -}; // DBMetaImpl +}; // DBMetaImpl -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 22c282d4ea..bced85af6d 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -17,25 +17,25 @@ #include #include -#include -#include #include #include +#include +#include -#include "utils/easylogging++.h" -#include "utils/SignalUtil.h" -#include "utils/CommonUtil.h" +#include "../version.h" #include "metrics/Metrics.h" #include "server/Server.h" -#include "../version.h" - +#include "utils/CommonUtil.h" +#include "utils/SignalUtil.h" +#include "utils/easylogging++.h" INITIALIZE_EASYLOGGINGPP -void print_help(const std::string &app_name); +void +print_help(const std::string& app_name); int -main(int argc, char *argv[]) { +main(int argc, char* argv[]) { std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl; std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << " built at " << BUILD_TIME << std::endl; @@ -63,21 +63,21 @@ main(int argc, char *argv[]) { while ((value = getopt_long(argc, argv, "c:l:p:dh", long_options, &option_index)) != -1) { switch (value) { case 'c': { - char *config_filename_ptr = strdup(optarg); + char* config_filename_ptr = strdup(optarg); config_filename = config_filename_ptr; free(config_filename_ptr); std::cout << "Loading configuration from: " << config_filename << std::endl; break; } case 'l': { - char *log_filename_ptr = strdup(optarg); + char* log_filename_ptr = strdup(optarg); log_config_file = log_filename_ptr; free(log_filename_ptr); std::cout << "Initial log config from: " << log_config_file << std::endl; break; } case 'p': { - char *pid_filename_ptr = strdup(optarg); + char* pid_filename_ptr = strdup(optarg); pid_filename = pid_filename_ptr; free(pid_filename_ptr); std::cout << pid_filename << std::endl; @@ -106,7 +106,7 @@ main(int argc, char *argv[]) { signal(SIGUSR2, zilliz::milvus::server::SignalUtil::HandleSignal); signal(SIGTERM, zilliz::milvus::server::SignalUtil::HandleSignal); - zilliz::milvus::server::Server &server = zilliz::milvus::server::Server::GetInstance(); + zilliz::milvus::server::Server& server = zilliz::milvus::server::Server::GetInstance(); server.Init(start_daemonized, pid_filename, config_filename, log_config_file); server.Start(); @@ -117,7 +117,7 @@ main(int argc, char *argv[]) { } void -print_help(const std::string &app_name) { +print_help(const std::string& app_name) { std::cout << std::endl << "Usage: " << app_name << " [OPTIONS]" << std::endl << std::endl; std::cout << " Options:" << std::endl; std::cout << " -h --help Print this help" << std::endl; diff --git a/cpp/src/metrics/MetricBase.h b/cpp/src/metrics/MetricBase.h index 32a0e715e6..5442b9b714 100644 --- a/cpp/src/metrics/MetricBase.h +++ b/cpp/src/metrics/MetricBase.h @@ -15,11 +15,10 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "utils/Error.h" #include "SystemInfo.h" +#include "utils/Error.h" #include @@ -28,142 +27,185 @@ namespace milvus { namespace server { class MetricsBase { public: - static MetricsBase & + static MetricsBase& GetInstance() { static MetricsBase instance; return instance; } - virtual ErrorCode Init() { + virtual ErrorCode + Init() { } - virtual void AddVectorsSuccessTotalIncrement(double value = 1) { + virtual void + AddVectorsSuccessTotalIncrement(double value = 1) { } - virtual void AddVectorsFailTotalIncrement(double value = 1) { + virtual void + AddVectorsFailTotalIncrement(double value = 1) { } - virtual void AddVectorsDurationHistogramOberve(double value) { + virtual void + AddVectorsDurationHistogramOberve(double value) { } - virtual void RawFileSizeHistogramObserve(double value) { + virtual void + RawFileSizeHistogramObserve(double value) { } - virtual void IndexFileSizeHistogramObserve(double value) { + virtual void + IndexFileSizeHistogramObserve(double value) { } - virtual void BuildIndexDurationSecondsHistogramObserve(double value) { + virtual void + BuildIndexDurationSecondsHistogramObserve(double value) { } - virtual void CpuCacheUsageGaugeSet(double value) { + virtual void + CpuCacheUsageGaugeSet(double value) { } - virtual void GpuCacheUsageGaugeSet() { + virtual void + GpuCacheUsageGaugeSet() { } - virtual void MetaAccessTotalIncrement(double value = 1) { + virtual void + MetaAccessTotalIncrement(double value = 1) { } - virtual void MetaAccessDurationSecondsHistogramObserve(double value) { + virtual void + MetaAccessDurationSecondsHistogramObserve(double value) { } - virtual void FaissDiskLoadDurationSecondsHistogramObserve(double value) { + virtual void + FaissDiskLoadDurationSecondsHistogramObserve(double value) { } - virtual void FaissDiskLoadSizeBytesHistogramObserve(double value) { + virtual void + FaissDiskLoadSizeBytesHistogramObserve(double value) { } - virtual void CacheAccessTotalIncrement(double value = 1) { + virtual void + CacheAccessTotalIncrement(double value = 1) { } - virtual void MemTableMergeDurationSecondsHistogramObserve(double value) { + virtual void + MemTableMergeDurationSecondsHistogramObserve(double value) { } - virtual void SearchIndexDataDurationSecondsHistogramObserve(double value) { + virtual void + SearchIndexDataDurationSecondsHistogramObserve(double value) { } - virtual void SearchRawDataDurationSecondsHistogramObserve(double value) { + virtual void + SearchRawDataDurationSecondsHistogramObserve(double value) { } - virtual void IndexFileSizeTotalIncrement(double value = 1) { + virtual void + IndexFileSizeTotalIncrement(double value = 1) { } - virtual void RawFileSizeTotalIncrement(double value = 1) { + virtual void + RawFileSizeTotalIncrement(double value = 1) { } - virtual void IndexFileSizeGaugeSet(double value) { + virtual void + IndexFileSizeGaugeSet(double value) { } - virtual void RawFileSizeGaugeSet(double value) { + virtual void + RawFileSizeGaugeSet(double value) { } - virtual void FaissDiskLoadIOSpeedGaugeSet(double value) { + virtual void + FaissDiskLoadIOSpeedGaugeSet(double value) { } - virtual void QueryResponseSummaryObserve(double value) { + virtual void + QueryResponseSummaryObserve(double value) { } - virtual void DiskStoreIOSpeedGaugeSet(double value) { + virtual void + DiskStoreIOSpeedGaugeSet(double value) { } - virtual void DataFileSizeGaugeSet(double value) { + virtual void + DataFileSizeGaugeSet(double value) { } - virtual void AddVectorsSuccessGaugeSet(double value) { + virtual void + AddVectorsSuccessGaugeSet(double value) { } - virtual void AddVectorsFailGaugeSet(double value) { + virtual void + AddVectorsFailGaugeSet(double value) { } - virtual void QueryVectorResponseSummaryObserve(double value, int count = 1) { + virtual void + QueryVectorResponseSummaryObserve(double value, int count = 1) { } - virtual void QueryVectorResponsePerSecondGaugeSet(double value) { + virtual void + QueryVectorResponsePerSecondGaugeSet(double value) { } - virtual void CPUUsagePercentSet() { + virtual void + CPUUsagePercentSet() { } - virtual void RAMUsagePercentSet() { + virtual void + RAMUsagePercentSet() { } - virtual void QueryResponsePerSecondGaugeSet(double value) { + virtual void + QueryResponsePerSecondGaugeSet(double value) { } - virtual void GPUPercentGaugeSet() { + virtual void + GPUPercentGaugeSet() { } - virtual void GPUMemoryUsageGaugeSet() { + virtual void + GPUMemoryUsageGaugeSet() { } - virtual void AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) { + virtual void + AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) { } - virtual void QueryIndexTypePerSecondSet(std::string type, double value) { + virtual void + QueryIndexTypePerSecondSet(std::string type, double value) { } - virtual void ConnectionGaugeIncrement() { + virtual void + ConnectionGaugeIncrement() { } - virtual void ConnectionGaugeDecrement() { + virtual void + ConnectionGaugeDecrement() { } - virtual void KeepingAliveCounterIncrement(double value = 1) { + virtual void + KeepingAliveCounterIncrement(double value = 1) { } - virtual void OctetsSet() { + virtual void + OctetsSet() { } - virtual void CPUCoreUsagePercentSet() { + virtual void + CPUCoreUsagePercentSet() { } - virtual void GPUTemperature() { + virtual void + GPUTemperature() { } - virtual void CPUTemperature() { + virtual void + CPUTemperature() { } }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/Metrics.cpp b/cpp/src/metrics/Metrics.cpp index 0a1b333b42..9a182ce571 100644 --- a/cpp/src/metrics/Metrics.cpp +++ b/cpp/src/metrics/Metrics.cpp @@ -16,8 +16,8 @@ // under the License. #include "metrics/Metrics.h" -#include "server/Config.h" #include "PrometheusMetrics.h" +#include "server/Config.h" #include @@ -25,15 +25,15 @@ namespace zilliz { namespace milvus { namespace server { -MetricsBase & +MetricsBase& Metrics::GetInstance() { - static MetricsBase &instance = CreateMetricsCollector(); + static MetricsBase& instance = CreateMetricsCollector(); return instance; } -MetricsBase & +MetricsBase& Metrics::CreateMetricsCollector() { - Config &config = Config::GetInstance(); + Config& config = Config::GetInstance(); std::string collector_type_str; config.GetMetricConfigCollector(collector_type_str); @@ -45,6 +45,6 @@ Metrics::CreateMetricsCollector() { } } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/Metrics.h b/cpp/src/metrics/Metrics.h index 66e06b8ba1..f644004f14 100644 --- a/cpp/src/metrics/Metrics.h +++ b/cpp/src/metrics/Metrics.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "MetricBase.h" @@ -26,20 +25,18 @@ namespace milvus { namespace server { #define METRICS_NOW_TIME std::chrono::system_clock::now() -#define METRICS_MICROSECONDS(a, b) (std::chrono::duration_cast (b-a)).count(); +#define METRICS_MICROSECONDS(a, b) (std::chrono::duration_cast(b - a)).count(); -enum class MetricCollectorType { - INVALID, - PROMETHEUS, - ZABBIX -}; +enum class MetricCollectorType { INVALID, PROMETHEUS, ZABBIX }; class Metrics { public: - static MetricsBase &GetInstance(); + static MetricsBase& + GetInstance(); private: - static MetricsBase &CreateMetricsCollector(); + static MetricsBase& + CreateMetricsCollector(); }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CollectMetricsBase { @@ -50,7 +47,8 @@ class CollectMetricsBase { virtual ~CollectMetricsBase() = default; - double TimeFromBegine() { + double + TimeFromBegine() { auto end_time = METRICS_NOW_TIME; return METRICS_MICROSECONDS(start_time_, end_time); } @@ -63,7 +61,7 @@ class CollectMetricsBase { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CollectInsertMetrics : CollectMetricsBase { public: - CollectInsertMetrics(size_t n, Status &status) : n_(n), status_(status) { + CollectInsertMetrics(size_t n, Status& status) : n_(n), status_(status) { } ~CollectInsertMetrics() { @@ -87,7 +85,7 @@ class CollectInsertMetrics : CollectMetricsBase { private: size_t n_; - Status &status_; + Status& status_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -162,7 +160,7 @@ class CollectSerializeMetrics : CollectMetricsBase { ~CollectSerializeMetrics() { auto total_time = TimeFromBegine(); - server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double) size_ / total_time); + server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double)size_ / total_time); } private: @@ -177,8 +175,7 @@ class CollectAddMetrics : CollectMetricsBase { ~CollectAddMetrics() { auto total_time = TimeFromBegine(); - server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast(n_), - static_cast(dimension_), + server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast(n_), static_cast(dimension_), total_time); } @@ -256,6 +253,6 @@ class MetricCollector : CollectMetricsBase { } }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index 8dcc97aa8b..dd6410b46a 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. - #include "metrics/PrometheusMetrics.h" +#include "SystemInfo.h" #include "cache/GpuCacheMgr.h" #include "server/Config.h" #include "utils/Log.h" -#include "SystemInfo.h" #include #include @@ -32,7 +31,7 @@ namespace server { ErrorCode PrometheusMetrics::Init() { try { - Config &config = Config::GetInstance(); + Config& config = Config::GetInstance(); Status s = config.GetMetricConfigEnableMonitor(startup_); if (!s.ok()) return s.code(); if (!startup_) return SERVER_SUCCESS; @@ -49,7 +48,7 @@ PrometheusMetrics::Init() { // Exposer Registry exposer_ptr_->RegisterCollectable(registry_); - } catch (std::exception &ex) { + } catch (std::exception& ex) { SERVER_LOG_ERROR << "Failed to connect prometheus server: " << std::string(ex.what()); return SERVER_UNEXPECTED_ERROR; } @@ -79,8 +78,8 @@ PrometheusMetrics::GPUPercentGaugeSet() { std::vector used_memory = server::SystemInfo::GetInstance().GPUMemoryUsed(); for (int i = 0; i < numDevice; ++i) { - prometheus::Gauge &GPU_percent = GPU_percent_.Add({{"DeviceNum", std::to_string(i)}}); - double percent = (double) used_memory[i] / (double) used_total[i]; + prometheus::Gauge& GPU_percent = GPU_percent_.Add({{"DeviceNum", std::to_string(i)}}); + double percent = (double)used_memory[i] / (double)used_total[i]; GPU_percent.Set(percent * 100); } } @@ -93,7 +92,7 @@ PrometheusMetrics::GPUMemoryUsageGaugeSet() { int numDevice = server::SystemInfo::GetInstance().num_device(); for (int i = 0; i < numDevice; ++i) { - prometheus::Gauge &GPU_memory = GPU_memory_usage_.Add({{"DeviceNum", std::to_string(i)}}); + prometheus::Gauge& GPU_memory = GPU_memory_usage_.Add({{"DeviceNum", std::to_string(i)}}); GPU_memory.Set(values[i] / MtoB); } } @@ -155,54 +154,51 @@ PrometheusMetrics::OctetsSet() { void PrometheusMetrics::CPUCoreUsagePercentSet() { - if (!startup_) - return; + if (!startup_) return; std::vector cpu_core_percent = server::SystemInfo::GetInstance().CPUCorePercent(); for (int i = 0; i < cpu_core_percent.size(); ++i) { - prometheus::Gauge &core_percent = CPU_.Add({{"CPU", std::to_string(i)}}); + prometheus::Gauge& core_percent = CPU_.Add({{"CPU", std::to_string(i)}}); core_percent.Set(cpu_core_percent[i]); } } void PrometheusMetrics::GPUTemperature() { - if (!startup_) - return; + if (!startup_) return; std::vector GPU_temperatures = server::SystemInfo::GetInstance().GPUTemperature(); for (int i = 0; i < GPU_temperatures.size(); ++i) { - prometheus::Gauge &gpu_temp = GPU_temperature_.Add({{"GPU", std::to_string(i)}}); + prometheus::Gauge& gpu_temp = GPU_temperature_.Add({{"GPU", std::to_string(i)}}); gpu_temp.Set(GPU_temperatures[i]); } } void PrometheusMetrics::CPUTemperature() { - if (!startup_) - return; + if (!startup_) return; std::vector CPU_temperatures = server::SystemInfo::GetInstance().CPUTemperature(); for (int i = 0; i < CPU_temperatures.size(); ++i) { - prometheus::Gauge &cpu_temp = CPU_temperature_.Add({{"CPU", std::to_string(i)}}); + prometheus::Gauge& cpu_temp = CPU_temperature_.Add({{"CPU", std::to_string(i)}}); cpu_temp.Set(CPU_temperatures[i]); } } void PrometheusMetrics::GpuCacheUsageGaugeSet() { -// std::vector gpu_ids = {0}; -// for(auto i = 0; i < gpu_ids.size(); ++i) { -// uint64_t cache_usage = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheUsage(); -// uint64_t cache_capacity = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheCapacity(); -// prometheus::Gauge &gpu_cache = gpu_cache_usage_.Add({{"GPU_Cache", std::to_string(i)}}); -// gpu_cache.Set(cache_usage * 100 / cache_capacity); -// } + // std::vector gpu_ids = {0}; + // for(auto i = 0; i < gpu_ids.size(); ++i) { + // uint64_t cache_usage = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheUsage(); + // uint64_t cache_capacity = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheCapacity(); + // prometheus::Gauge &gpu_cache = gpu_cache_usage_.Add({{"GPU_Cache", std::to_string(i)}}); + // gpu_cache.Set(cache_usage * 100 / cache_capacity); + // } } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/PrometheusMetrics.h b/cpp/src/metrics/PrometheusMetrics.h index ce45c5e711..9e009186d2 100644 --- a/cpp/src/metrics/PrometheusMetrics.h +++ b/cpp/src/metrics/PrometheusMetrics.h @@ -17,19 +17,19 @@ #pragma once -#include -#include -#include #include +#include #include +#include #include +#include -#include "utils/Error.h" #include "MetricBase.h" +#include "utils/Error.h" #define METRICS_NOW_TIME std::chrono::system_clock::now() //#define server::Metrics::GetInstance() server::GetInstance() -#define METRICS_MICROSECONDS(a, b) (std::chrono::duration_cast (b-a)).count(); +#define METRICS_MICROSECONDS(a, b) (std::chrono::duration_cast(b - a)).count(); namespace zilliz { namespace milvus { @@ -37,13 +37,14 @@ namespace server { class PrometheusMetrics : public MetricsBase { public: - static PrometheusMetrics & + static PrometheusMetrics& GetInstance() { static PrometheusMetrics instance; return instance; } - ErrorCode Init(); + ErrorCode + Init(); private: std::shared_ptr exposer_ptr_; @@ -51,163 +52,191 @@ class PrometheusMetrics : public MetricsBase { bool startup_ = false; public: - void SetStartup(bool startup) { + void + SetStartup(bool startup) { startup_ = startup; } - void AddVectorsSuccessTotalIncrement(double value = 1.0) override { + void + AddVectorsSuccessTotalIncrement(double value = 1.0) override { if (startup_) { add_vectors_success_total_.Increment(value); } } - void AddVectorsFailTotalIncrement(double value = 1.0) override { + void + AddVectorsFailTotalIncrement(double value = 1.0) override { if (startup_) { add_vectors_fail_total_.Increment(value); } } - void AddVectorsDurationHistogramOberve(double value) override { + void + AddVectorsDurationHistogramOberve(double value) override { if (startup_) { add_vectors_duration_histogram_.Observe(value); } } - void RawFileSizeHistogramObserve(double value) override { + void + RawFileSizeHistogramObserve(double value) override { if (startup_) { raw_files_size_histogram_.Observe(value); } } - void IndexFileSizeHistogramObserve(double value) override { + void + IndexFileSizeHistogramObserve(double value) override { if (startup_) { index_files_size_histogram_.Observe(value); } } - void BuildIndexDurationSecondsHistogramObserve(double value) override { + void + BuildIndexDurationSecondsHistogramObserve(double value) override { if (startup_) { build_index_duration_seconds_histogram_.Observe(value); } } - void CpuCacheUsageGaugeSet(double value) override { + void + CpuCacheUsageGaugeSet(double value) override { if (startup_) { cpu_cache_usage_gauge_.Set(value); } } - void GpuCacheUsageGaugeSet() override; + void + GpuCacheUsageGaugeSet() override; - void MetaAccessTotalIncrement(double value = 1) override { + void + MetaAccessTotalIncrement(double value = 1) override { if (startup_) { meta_access_total_.Increment(value); } } - void MetaAccessDurationSecondsHistogramObserve(double value) override { + void + MetaAccessDurationSecondsHistogramObserve(double value) override { if (startup_) { meta_access_duration_seconds_histogram_.Observe(value); } } - void FaissDiskLoadDurationSecondsHistogramObserve(double value) override { + void + FaissDiskLoadDurationSecondsHistogramObserve(double value) override { if (startup_) { faiss_disk_load_duration_seconds_histogram_.Observe(value); } } - void FaissDiskLoadSizeBytesHistogramObserve(double value) override { + void + FaissDiskLoadSizeBytesHistogramObserve(double value) override { if (startup_) { faiss_disk_load_size_bytes_histogram_.Observe(value); } } - void FaissDiskLoadIOSpeedGaugeSet(double value) override { + void + FaissDiskLoadIOSpeedGaugeSet(double value) override { if (startup_) { faiss_disk_load_IO_speed_gauge_.Set(value); } } - void CacheAccessTotalIncrement(double value = 1) override { + void + CacheAccessTotalIncrement(double value = 1) override { if (startup_) { cache_access_total_.Increment(value); } } - void MemTableMergeDurationSecondsHistogramObserve(double value) override { + void + MemTableMergeDurationSecondsHistogramObserve(double value) override { if (startup_) { mem_table_merge_duration_seconds_histogram_.Observe(value); } } - void SearchIndexDataDurationSecondsHistogramObserve(double value) override { + void + SearchIndexDataDurationSecondsHistogramObserve(double value) override { if (startup_) { search_index_data_duration_seconds_histogram_.Observe(value); } } - void SearchRawDataDurationSecondsHistogramObserve(double value) override { + void + SearchRawDataDurationSecondsHistogramObserve(double value) override { if (startup_) { search_raw_data_duration_seconds_histogram_.Observe(value); } } - void IndexFileSizeTotalIncrement(double value = 1) override { + void + IndexFileSizeTotalIncrement(double value = 1) override { if (startup_) { index_file_size_total_.Increment(value); } } - void RawFileSizeTotalIncrement(double value = 1) override { + void + RawFileSizeTotalIncrement(double value = 1) override { if (startup_) { raw_file_size_total_.Increment(value); } } - void IndexFileSizeGaugeSet(double value) override { + void + IndexFileSizeGaugeSet(double value) override { if (startup_) { index_file_size_gauge_.Set(value); } } - void RawFileSizeGaugeSet(double value) override { + void + RawFileSizeGaugeSet(double value) override { if (startup_) { raw_file_size_gauge_.Set(value); } } - void QueryResponseSummaryObserve(double value) override { + void + QueryResponseSummaryObserve(double value) override { if (startup_) { query_response_summary_.Observe(value); } } - void DiskStoreIOSpeedGaugeSet(double value) override { + void + DiskStoreIOSpeedGaugeSet(double value) override { if (startup_) { disk_store_IO_speed_gauge_.Set(value); } } - void DataFileSizeGaugeSet(double value) override { + void + DataFileSizeGaugeSet(double value) override { if (startup_) { data_file_size_gauge_.Set(value); } } - void AddVectorsSuccessGaugeSet(double value) override { + void + AddVectorsSuccessGaugeSet(double value) override { if (startup_) { add_vectors_success_gauge_.Set(value); } } - void AddVectorsFailGaugeSet(double value) override { + void + AddVectorsFailGaugeSet(double value) override { if (startup_) { add_vectors_fail_gauge_.Set(value); } } - void QueryVectorResponseSummaryObserve(double value, int count = 1) override { + void + QueryVectorResponseSummaryObserve(double value, int count = 1) override { if (startup_) { for (int i = 0; i < count; ++i) { query_vector_response_summary_.Observe(value); @@ -215,412 +244,414 @@ class PrometheusMetrics : public MetricsBase { } } - void QueryVectorResponsePerSecondGaugeSet(double value) override { + void + QueryVectorResponsePerSecondGaugeSet(double value) override { if (startup_) { query_vector_response_per_second_gauge_.Set(value); } } - void CPUUsagePercentSet() override; - void CPUCoreUsagePercentSet() override; + void + CPUUsagePercentSet() override; + void + CPUCoreUsagePercentSet() override; - void RAMUsagePercentSet() override; + void + RAMUsagePercentSet() override; - void QueryResponsePerSecondGaugeSet(double value) override { + void + QueryResponsePerSecondGaugeSet(double value) override { if (startup_) { query_response_per_second_gauge.Set(value); } } - void GPUPercentGaugeSet() override; - void GPUMemoryUsageGaugeSet() override; - void AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) override; - void QueryIndexTypePerSecondSet(std::string type, double value) override; - void ConnectionGaugeIncrement() override; - void ConnectionGaugeDecrement() override; + void + GPUPercentGaugeSet() override; + void + GPUMemoryUsageGaugeSet() override; + void + AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) override; + void + QueryIndexTypePerSecondSet(std::string type, double value) override; + void + ConnectionGaugeIncrement() override; + void + ConnectionGaugeDecrement() override; - void KeepingAliveCounterIncrement(double value = 1) override { + void + KeepingAliveCounterIncrement(double value = 1) override { if (startup_) { keeping_alive_counter_.Increment(value); } } - void OctetsSet() override; + void + OctetsSet() override; - void GPUTemperature() override; - void CPUTemperature() override; + void + GPUTemperature() override; + void + CPUTemperature() override; - std::shared_ptr &exposer_ptr() { + std::shared_ptr& + exposer_ptr() { return exposer_ptr_; } -// prometheus::Exposer& exposer() { return exposer_;} - std::shared_ptr ®istry_ptr() { + // prometheus::Exposer& exposer() { return exposer_;} + std::shared_ptr& + registry_ptr() { return registry_; } // ..... private: ////all from db_connection.cpp -// prometheus::Family &connect_request_ = prometheus::BuildCounter() -// .Name("connection_total") -// .Help("total number of connection has been made") -// .Register(*registry_); -// prometheus::Counter &connection_total_ = connect_request_.Add({}); + // prometheus::Family &connect_request_ = prometheus::BuildCounter() + // .Name("connection_total") + // .Help("total number of connection has been made") + // .Register(*registry_); + // prometheus::Counter &connection_total_ = connect_request_.Add({}); ////all from DBImpl.cpp using BucketBoundaries = std::vector; - //record add_group request - prometheus::Family &add_group_request_ = prometheus::BuildCounter() - .Name("add_group_request_total") - .Help("the number of add_group request") - .Register(*registry_); + // record add_group request + prometheus::Family& add_group_request_ = prometheus::BuildCounter() + .Name("add_group_request_total") + .Help("the number of add_group request") + .Register(*registry_); - prometheus::Counter &add_group_success_total_ = add_group_request_.Add({{"outcome", "success"}}); - prometheus::Counter &add_group_fail_total_ = add_group_request_.Add({{"outcome", "fail"}}); + prometheus::Counter& add_group_success_total_ = add_group_request_.Add({{"outcome", "success"}}); + prometheus::Counter& add_group_fail_total_ = add_group_request_.Add({{"outcome", "fail"}}); - //record get_group request - prometheus::Family &get_group_request_ = prometheus::BuildCounter() - .Name("get_group_request_total") - .Help("the number of get_group request") - .Register(*registry_); + // record get_group request + prometheus::Family& get_group_request_ = prometheus::BuildCounter() + .Name("get_group_request_total") + .Help("the number of get_group request") + .Register(*registry_); - prometheus::Counter &get_group_success_total_ = get_group_request_.Add({{"outcome", "success"}}); - prometheus::Counter &get_group_fail_total_ = get_group_request_.Add({{"outcome", "fail"}}); + prometheus::Counter& get_group_success_total_ = get_group_request_.Add({{"outcome", "success"}}); + prometheus::Counter& get_group_fail_total_ = get_group_request_.Add({{"outcome", "fail"}}); - //record has_group request - prometheus::Family &has_group_request_ = prometheus::BuildCounter() - .Name("has_group_request_total") - .Help("the number of has_group request") - .Register(*registry_); + // record has_group request + prometheus::Family& has_group_request_ = prometheus::BuildCounter() + .Name("has_group_request_total") + .Help("the number of has_group request") + .Register(*registry_); - prometheus::Counter &has_group_success_total_ = has_group_request_.Add({{"outcome", "success"}}); - prometheus::Counter &has_group_fail_total_ = has_group_request_.Add({{"outcome", "fail"}}); + prometheus::Counter& has_group_success_total_ = has_group_request_.Add({{"outcome", "success"}}); + prometheus::Counter& has_group_fail_total_ = has_group_request_.Add({{"outcome", "fail"}}); - //record get_group_files - prometheus::Family &get_group_files_request_ = prometheus::BuildCounter() - .Name("get_group_files_request_total") - .Help("the number of get_group_files request") - .Register(*registry_); + // record get_group_files + prometheus::Family& get_group_files_request_ = + prometheus::BuildCounter() + .Name("get_group_files_request_total") + .Help("the number of get_group_files request") + .Register(*registry_); - prometheus::Counter &get_group_files_success_total_ = get_group_files_request_.Add({{"outcome", "success"}}); - prometheus::Counter &get_group_files_fail_total_ = get_group_files_request_.Add({{"outcome", "fail"}}); + prometheus::Counter& get_group_files_success_total_ = get_group_files_request_.Add({{"outcome", "success"}}); + prometheus::Counter& get_group_files_fail_total_ = get_group_files_request_.Add({{"outcome", "fail"}}); - //record add_vectors count and average time - //need to be considered - prometheus::Family &add_vectors_request_ = prometheus::BuildCounter() - .Name("add_vectors_request_total") - .Help("the number of vectors added") - .Register(*registry_); - prometheus::Counter &add_vectors_success_total_ = add_vectors_request_.Add({{"outcome", "success"}}); - prometheus::Counter &add_vectors_fail_total_ = add_vectors_request_.Add({{"outcome", "fail"}}); + // record add_vectors count and average time + // need to be considered + prometheus::Family& add_vectors_request_ = prometheus::BuildCounter() + .Name("add_vectors_request_total") + .Help("the number of vectors added") + .Register(*registry_); + prometheus::Counter& add_vectors_success_total_ = add_vectors_request_.Add({{"outcome", "success"}}); + prometheus::Counter& add_vectors_fail_total_ = add_vectors_request_.Add({{"outcome", "fail"}}); - prometheus::Family &add_vectors_duration_seconds_ = prometheus::BuildHistogram() - .Name("add_vector_duration_microseconds") - .Help("average time of adding every vector") - .Register(*registry_); - prometheus::Histogram &add_vectors_duration_histogram_ = + prometheus::Family& add_vectors_duration_seconds_ = + prometheus::BuildHistogram() + .Name("add_vector_duration_microseconds") + .Help("average time of adding every vector") + .Register(*registry_); + prometheus::Histogram& add_vectors_duration_histogram_ = add_vectors_duration_seconds_.Add({}, BucketBoundaries{0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.08, 0.1, 0.5, 1}); - //record search count and average time - prometheus::Family &search_request_ = prometheus::BuildCounter() - .Name("search_request_total") - .Help("the number of search request") - .Register(*registry_); - prometheus::Counter &search_success_total_ = search_request_.Add({{"outcome", "success"}}); - prometheus::Counter &search_fail_total_ = search_request_.Add({{"outcome", "fail"}}); + // record search count and average time + prometheus::Family& search_request_ = prometheus::BuildCounter() + .Name("search_request_total") + .Help("the number of search request") + .Register(*registry_); + prometheus::Counter& search_success_total_ = search_request_.Add({{"outcome", "success"}}); + prometheus::Counter& search_fail_total_ = search_request_.Add({{"outcome", "fail"}}); - prometheus::Family &search_request_duration_seconds_ = prometheus::BuildHistogram() - .Name("search_request_duration_microsecond") - .Help("histogram of processing time for each search") - .Register(*registry_); - prometheus::Histogram - &search_duration_histogram_ = search_request_duration_seconds_.Add({}, BucketBoundaries{0.1, 1.0, 10.0}); + prometheus::Family& search_request_duration_seconds_ = + prometheus::BuildHistogram() + .Name("search_request_duration_microsecond") + .Help("histogram of processing time for each search") + .Register(*registry_); + prometheus::Histogram& search_duration_histogram_ = + search_request_duration_seconds_.Add({}, BucketBoundaries{0.1, 1.0, 10.0}); - //record raw_files size histogram - prometheus::Family &raw_files_size_ = prometheus::BuildHistogram() - .Name("search_raw_files_bytes") - .Help("histogram of raw files size by bytes") - .Register(*registry_); - prometheus::Histogram - &raw_files_size_histogram_ = raw_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10}); + // record raw_files size histogram + prometheus::Family& raw_files_size_ = prometheus::BuildHistogram() + .Name("search_raw_files_bytes") + .Help("histogram of raw files size by bytes") + .Register(*registry_); + prometheus::Histogram& raw_files_size_histogram_ = + raw_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10}); - //record index_files size histogram - prometheus::Family &index_files_size_ = prometheus::BuildHistogram() - .Name("search_index_files_bytes") - .Help("histogram of index files size by bytes") - .Register(*registry_); - prometheus::Histogram - &index_files_size_histogram_ = index_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10}); + // record index_files size histogram + prometheus::Family& index_files_size_ = prometheus::BuildHistogram() + .Name("search_index_files_bytes") + .Help("histogram of index files size by bytes") + .Register(*registry_); + prometheus::Histogram& index_files_size_histogram_ = + index_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10}); - //record index and raw files size counter - prometheus::Family &file_size_total_ = prometheus::BuildCounter() - .Name("search_file_size_total") - .Help("searched index and raw file size") - .Register(*registry_); - prometheus::Counter &index_file_size_total_ = file_size_total_.Add({{"type", "index"}}); - prometheus::Counter &raw_file_size_total_ = file_size_total_.Add({{"type", "raw"}}); + // record index and raw files size counter + prometheus::Family& file_size_total_ = prometheus::BuildCounter() + .Name("search_file_size_total") + .Help("searched index and raw file size") + .Register(*registry_); + prometheus::Counter& index_file_size_total_ = file_size_total_.Add({{"type", "index"}}); + prometheus::Counter& raw_file_size_total_ = file_size_total_.Add({{"type", "raw"}}); - //record index and raw files size counter - prometheus::Family &file_size_gauge_ = prometheus::BuildGauge() - .Name("search_file_size_gauge") - .Help("searched current index and raw file size") - .Register(*registry_); - prometheus::Gauge &index_file_size_gauge_ = file_size_gauge_.Add({{"type", "index"}}); - prometheus::Gauge &raw_file_size_gauge_ = file_size_gauge_.Add({{"type", "raw"}}); + // record index and raw files size counter + prometheus::Family& file_size_gauge_ = prometheus::BuildGauge() + .Name("search_file_size_gauge") + .Help("searched current index and raw file size") + .Register(*registry_); + prometheus::Gauge& index_file_size_gauge_ = file_size_gauge_.Add({{"type", "index"}}); + prometheus::Gauge& raw_file_size_gauge_ = file_size_gauge_.Add({{"type", "raw"}}); - //record processing time for building index - prometheus::Family &build_index_duration_seconds_ = prometheus::BuildHistogram() - .Name("build_index_duration_microseconds") - .Help("histogram of processing time for building index") - .Register(*registry_); - prometheus::Histogram &build_index_duration_seconds_histogram_ = + // record processing time for building index + prometheus::Family& build_index_duration_seconds_ = + prometheus::BuildHistogram() + .Name("build_index_duration_microseconds") + .Help("histogram of processing time for building index") + .Register(*registry_); + prometheus::Histogram& build_index_duration_seconds_histogram_ = build_index_duration_seconds_.Add({}, BucketBoundaries{5e5, 2e6, 4e6, 6e6, 8e6, 1e7}); - //record processing time for all building index - prometheus::Family &all_build_index_duration_seconds_ = prometheus::BuildHistogram() - .Name("all_build_index_duration_microseconds") - .Help("histogram of processing time for building index") - .Register(*registry_); - prometheus::Histogram &all_build_index_duration_seconds_histogram_ = + // record processing time for all building index + prometheus::Family& all_build_index_duration_seconds_ = + prometheus::BuildHistogram() + .Name("all_build_index_duration_microseconds") + .Help("histogram of processing time for building index") + .Register(*registry_); + prometheus::Histogram& all_build_index_duration_seconds_histogram_ = all_build_index_duration_seconds_.Add({}, BucketBoundaries{2e6, 4e6, 6e6, 8e6, 1e7}); - //record duration of merging mem table - prometheus::Family &mem_table_merge_duration_seconds_ = prometheus::BuildHistogram() - .Name("mem_table_merge_duration_microseconds") - .Help("histogram of processing time for merging mem tables") - .Register(*registry_); - prometheus::Histogram &mem_table_merge_duration_seconds_histogram_ = + // record duration of merging mem table + prometheus::Family& mem_table_merge_duration_seconds_ = + prometheus::BuildHistogram() + .Name("mem_table_merge_duration_microseconds") + .Help("histogram of processing time for merging mem tables") + .Register(*registry_); + prometheus::Histogram& mem_table_merge_duration_seconds_histogram_ = mem_table_merge_duration_seconds_.Add({}, BucketBoundaries{5e4, 1e5, 2e5, 4e5, 6e5, 8e5, 1e6}); - //record search index and raw data duration - prometheus::Family &search_data_duration_seconds_ = prometheus::BuildHistogram() - .Name("search_data_duration_microseconds") - .Help("histograms of processing time for search index and raw data") - .Register(*registry_); - prometheus::Histogram &search_index_data_duration_seconds_histogram_ = + // record search index and raw data duration + prometheus::Family& search_data_duration_seconds_ = + prometheus::BuildHistogram() + .Name("search_data_duration_microseconds") + .Help("histograms of processing time for search index and raw data") + .Register(*registry_); + prometheus::Histogram& search_index_data_duration_seconds_histogram_ = search_data_duration_seconds_.Add({{"type", "index"}}, BucketBoundaries{1e5, 2e5, 4e5, 6e5, 8e5}); - prometheus::Histogram &search_raw_data_duration_seconds_histogram_ = + prometheus::Histogram& search_raw_data_duration_seconds_histogram_ = search_data_duration_seconds_.Add({{"type", "raw"}}, BucketBoundaries{1e5, 2e5, 4e5, 6e5, 8e5}); - ////all form Cache.cpp - //record cache usage, when insert/erase/clear/free - + // record cache usage, when insert/erase/clear/free ////all from Meta.cpp - //record meta visit count and time -// prometheus::Family &meta_visit_ = prometheus::BuildCounter() -// .Name("meta_visit_total") -// .Help("the number of accessing Meta") -// .Register(*registry_); -// prometheus::Counter &meta_visit_total_ = meta_visit_.Add({{}}); -// -// prometheus::Family &meta_visit_duration_seconds_ = prometheus::BuildHistogram() -// .Name("meta_visit_duration_seconds") -// .Help("histogram of processing time to get data from mata") -// .Register(*registry_); -// prometheus::Histogram &meta_visit_duration_seconds_histogram_ = -// meta_visit_duration_seconds_.Add({{}}, BucketBoundaries{0.1, 1.0, 10.0}); - + // record meta visit count and time + // prometheus::Family &meta_visit_ = prometheus::BuildCounter() + // .Name("meta_visit_total") + // .Help("the number of accessing Meta") + // .Register(*registry_); + // prometheus::Counter &meta_visit_total_ = meta_visit_.Add({{}}); + // + // prometheus::Family &meta_visit_duration_seconds_ = prometheus::BuildHistogram() + // .Name("meta_visit_duration_seconds") + // .Help("histogram of processing time to get data from mata") + // .Register(*registry_); + // prometheus::Histogram &meta_visit_duration_seconds_histogram_ = + // meta_visit_duration_seconds_.Add({{}}, BucketBoundaries{0.1, 1.0, 10.0}); ////all from MemManager.cpp - //record memory usage percent - prometheus::Family &mem_usage_percent_ = prometheus::BuildGauge() - .Name("memory_usage_percent") - .Help("memory usage percent") - .Register(*registry_); - prometheus::Gauge &mem_usage_percent_gauge_ = mem_usage_percent_.Add({}); + // record memory usage percent + prometheus::Family& mem_usage_percent_ = + prometheus::BuildGauge().Name("memory_usage_percent").Help("memory usage percent").Register(*registry_); + prometheus::Gauge& mem_usage_percent_gauge_ = mem_usage_percent_.Add({}); - //record memory usage toal - prometheus::Family &mem_usage_total_ = prometheus::BuildGauge() - .Name("memory_usage_total") - .Help("memory usage total") - .Register(*registry_); - prometheus::Gauge &mem_usage_total_gauge_ = mem_usage_total_.Add({}); + // record memory usage toal + prometheus::Family& mem_usage_total_ = + prometheus::BuildGauge().Name("memory_usage_total").Help("memory usage total").Register(*registry_); + prometheus::Gauge& mem_usage_total_gauge_ = mem_usage_total_.Add({}); ////all from DBMetaImpl.cpp - //record meta access count - prometheus::Family &meta_access_ = prometheus::BuildCounter() - .Name("meta_access_total") - .Help("the number of meta accessing") - .Register(*registry_); - prometheus::Counter &meta_access_total_ = meta_access_.Add({}); + // record meta access count + prometheus::Family& meta_access_ = + prometheus::BuildCounter().Name("meta_access_total").Help("the number of meta accessing").Register(*registry_); + prometheus::Counter& meta_access_total_ = meta_access_.Add({}); - //record meta access duration - prometheus::Family &meta_access_duration_seconds_ = prometheus::BuildHistogram() - .Name("meta_access_duration_microseconds") - .Help("histogram of processing time for accessing mata") - .Register(*registry_); - prometheus::Histogram &meta_access_duration_seconds_histogram_ = + // record meta access duration + prometheus::Family& meta_access_duration_seconds_ = + prometheus::BuildHistogram() + .Name("meta_access_duration_microseconds") + .Help("histogram of processing time for accessing mata") + .Register(*registry_); + prometheus::Histogram& meta_access_duration_seconds_histogram_ = meta_access_duration_seconds_.Add({}, BucketBoundaries{100, 300, 500, 700, 900, 2000, 4000, 6000, 8000, 20000}); ////all from FaissExecutionEngine.cpp - //record data loading from disk count, size, duration, IO speed - prometheus::Family &disk_load_duration_second_ = prometheus::BuildHistogram() - .Name("disk_load_duration_microseconds") - .Help("Histogram of processing time for loading data from disk") - .Register(*registry_); - prometheus::Histogram &faiss_disk_load_duration_seconds_histogram_ = + // record data loading from disk count, size, duration, IO speed + prometheus::Family& disk_load_duration_second_ = + prometheus::BuildHistogram() + .Name("disk_load_duration_microseconds") + .Help("Histogram of processing time for loading data from disk") + .Register(*registry_); + prometheus::Histogram& faiss_disk_load_duration_seconds_histogram_ = disk_load_duration_second_.Add({{"DB", "Faiss"}}, BucketBoundaries{2e5, 4e5, 6e5, 8e5}); - prometheus::Family &disk_load_size_bytes_ = prometheus::BuildHistogram() - .Name("disk_load_size_bytes") - .Help("Histogram of data size by bytes for loading data from disk") - .Register(*registry_); - prometheus::Histogram &faiss_disk_load_size_bytes_histogram_ = + prometheus::Family& disk_load_size_bytes_ = + prometheus::BuildHistogram() + .Name("disk_load_size_bytes") + .Help("Histogram of data size by bytes for loading data from disk") + .Register(*registry_); + prometheus::Histogram& faiss_disk_load_size_bytes_histogram_ = disk_load_size_bytes_.Add({{"DB", "Faiss"}}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9}); -// prometheus::Family &disk_load_IO_speed_ = prometheus::BuildHistogram() -// .Name("disk_load_IO_speed_byte_per_sec") -// .Help("Histogram of IO speed for loading data from disk") -// .Register(*registry_); -// prometheus::Histogram &faiss_disk_load_IO_speed_histogram_ = -// disk_load_IO_speed_.Add({{"DB","Faiss"}},BucketBoundaries{1000, 2000, 3000, 4000, 6000, 8000}); + // prometheus::Family &disk_load_IO_speed_ = prometheus::BuildHistogram() + // .Name("disk_load_IO_speed_byte_per_sec") + // .Help("Histogram of IO speed for loading data from disk") + // .Register(*registry_); + // prometheus::Histogram &faiss_disk_load_IO_speed_histogram_ = + // disk_load_IO_speed_.Add({{"DB","Faiss"}},BucketBoundaries{1000, 2000, 3000, 4000, 6000, 8000}); - prometheus::Family &faiss_disk_load_IO_speed_ = prometheus::BuildGauge() - .Name("disk_load_IO_speed_byte_per_microsec") - .Help("disk IO speed ") - .Register(*registry_); - prometheus::Gauge &faiss_disk_load_IO_speed_gauge_ = faiss_disk_load_IO_speed_.Add({{"DB", "Faiss"}}); + prometheus::Family& faiss_disk_load_IO_speed_ = prometheus::BuildGauge() + .Name("disk_load_IO_speed_byte_per_microsec") + .Help("disk IO speed ") + .Register(*registry_); + prometheus::Gauge& faiss_disk_load_IO_speed_gauge_ = faiss_disk_load_IO_speed_.Add({{"DB", "Faiss"}}); ////all from CacheMgr.cpp - //record cache access count - prometheus::Family &cache_access_ = prometheus::BuildCounter() - .Name("cache_access_total") - .Help("the count of accessing cache ") - .Register(*registry_); - prometheus::Counter &cache_access_total_ = cache_access_.Add({}); + // record cache access count + prometheus::Family& cache_access_ = prometheus::BuildCounter() + .Name("cache_access_total") + .Help("the count of accessing cache ") + .Register(*registry_); + prometheus::Counter& cache_access_total_ = cache_access_.Add({}); // record CPU cache usage and % - prometheus::Family &cpu_cache_usage_ = prometheus::BuildGauge() - .Name("cache_usage_bytes") - .Help("current cache usage by bytes") - .Register(*registry_); - prometheus::Gauge &cpu_cache_usage_gauge_ = cpu_cache_usage_.Add({}); + prometheus::Family& cpu_cache_usage_ = + prometheus::BuildGauge().Name("cache_usage_bytes").Help("current cache usage by bytes").Register(*registry_); + prometheus::Gauge& cpu_cache_usage_gauge_ = cpu_cache_usage_.Add({}); - //record GPU cache usage and % - prometheus::Family &gpu_cache_usage_ = prometheus::BuildGauge() - .Name("gpu_cache_usage_bytes") - .Help("current gpu cache usage by bytes") - .Register(*registry_); + // record GPU cache usage and % + prometheus::Family& gpu_cache_usage_ = prometheus::BuildGauge() + .Name("gpu_cache_usage_bytes") + .Help("current gpu cache usage by bytes") + .Register(*registry_); // record query response using Quantiles = std::vector; - prometheus::Family &query_response_ = prometheus::BuildSummary() - .Name("query_response_summary") - .Help("query response summary") - .Register(*registry_); - prometheus::Summary - &query_response_summary_ = query_response_.Add({}, Quantiles{{0.95, 0.00}, {0.9, 0.05}, {0.8, 0.1}}); + prometheus::Family& query_response_ = + prometheus::BuildSummary().Name("query_response_summary").Help("query response summary").Register(*registry_); + prometheus::Summary& query_response_summary_ = + query_response_.Add({}, Quantiles{{0.95, 0.00}, {0.9, 0.05}, {0.8, 0.1}}); - prometheus::Family &query_vector_response_ = prometheus::BuildSummary() - .Name("query_vector_response_summary") - .Help("query each vector response summary") - .Register(*registry_); - prometheus::Summary &query_vector_response_summary_ = + prometheus::Family& query_vector_response_ = prometheus::BuildSummary() + .Name("query_vector_response_summary") + .Help("query each vector response summary") + .Register(*registry_); + prometheus::Summary& query_vector_response_summary_ = query_vector_response_.Add({}, Quantiles{{0.95, 0.00}, {0.9, 0.05}, {0.8, 0.1}}); - prometheus::Family &query_vector_response_per_second_ = prometheus::BuildGauge() - .Name("query_vector_response_per_microsecond") - .Help("the number of vectors can be queried every second ") - .Register(*registry_); - prometheus::Gauge &query_vector_response_per_second_gauge_ = query_vector_response_per_second_.Add({}); + prometheus::Family& query_vector_response_per_second_ = + prometheus::BuildGauge() + .Name("query_vector_response_per_microsecond") + .Help("the number of vectors can be queried every second ") + .Register(*registry_); + prometheus::Gauge& query_vector_response_per_second_gauge_ = query_vector_response_per_second_.Add({}); - prometheus::Family &query_response_per_second_ = prometheus::BuildGauge() - .Name("query_response_per_microsecond") - .Help("the number of queries can be processed every microsecond") - .Register(*registry_); - prometheus::Gauge &query_response_per_second_gauge = query_response_per_second_.Add({}); + prometheus::Family& query_response_per_second_ = + prometheus::BuildGauge() + .Name("query_response_per_microsecond") + .Help("the number of queries can be processed every microsecond") + .Register(*registry_); + prometheus::Gauge& query_response_per_second_gauge = query_response_per_second_.Add({}); - prometheus::Family &disk_store_IO_speed_ = prometheus::BuildGauge() - .Name("disk_store_IO_speed_bytes_per_microseconds") - .Help("disk_store_IO_speed") - .Register(*registry_); - prometheus::Gauge &disk_store_IO_speed_gauge_ = disk_store_IO_speed_.Add({}); + prometheus::Family& disk_store_IO_speed_ = + prometheus::BuildGauge() + .Name("disk_store_IO_speed_bytes_per_microseconds") + .Help("disk_store_IO_speed") + .Register(*registry_); + prometheus::Gauge& disk_store_IO_speed_gauge_ = disk_store_IO_speed_.Add({}); - prometheus::Family &data_file_size_ = prometheus::BuildGauge() - .Name("data_file_size_bytes") - .Help("data file size by bytes") - .Register(*registry_); - prometheus::Gauge &data_file_size_gauge_ = data_file_size_.Add({}); + prometheus::Family& data_file_size_ = + prometheus::BuildGauge().Name("data_file_size_bytes").Help("data file size by bytes").Register(*registry_); + prometheus::Gauge& data_file_size_gauge_ = data_file_size_.Add({}); - prometheus::Family &add_vectors_ = prometheus::BuildGauge() - .Name("add_vectors") - .Help("current added vectors") - .Register(*registry_); - prometheus::Gauge &add_vectors_success_gauge_ = add_vectors_.Add({{"outcome", "success"}}); - prometheus::Gauge &add_vectors_fail_gauge_ = add_vectors_.Add({{"outcome", "fail"}}); + prometheus::Family& add_vectors_ = + prometheus::BuildGauge().Name("add_vectors").Help("current added vectors").Register(*registry_); + prometheus::Gauge& add_vectors_success_gauge_ = add_vectors_.Add({{"outcome", "success"}}); + prometheus::Gauge& add_vectors_fail_gauge_ = add_vectors_.Add({{"outcome", "fail"}}); - prometheus::Family &add_vectors_per_second_ = prometheus::BuildGauge() - .Name("add_vectors_throughput_per_microsecond") - .Help("add vectors throughput per microsecond") - .Register(*registry_); - prometheus::Gauge &add_vectors_per_second_gauge_ = add_vectors_per_second_.Add({}); + prometheus::Family& add_vectors_per_second_ = prometheus::BuildGauge() + .Name("add_vectors_throughput_per_microsecond") + .Help("add vectors throughput per microsecond") + .Register(*registry_); + prometheus::Gauge& add_vectors_per_second_gauge_ = add_vectors_per_second_.Add({}); - prometheus::Family &CPU_ = prometheus::BuildGauge() - .Name("CPU_usage_percent") - .Help("CPU usage percent by this this process") - .Register(*registry_); - prometheus::Gauge &CPU_usage_percent_ = CPU_.Add({{"CPU", "avg"}}); + prometheus::Family& CPU_ = prometheus::BuildGauge() + .Name("CPU_usage_percent") + .Help("CPU usage percent by this this process") + .Register(*registry_); + prometheus::Gauge& CPU_usage_percent_ = CPU_.Add({{"CPU", "avg"}}); - prometheus::Family &RAM_ = prometheus::BuildGauge() - .Name("RAM_usage_percent") - .Help("RAM usage percent by this process") - .Register(*registry_); - prometheus::Gauge &RAM_usage_percent_ = RAM_.Add({}); + prometheus::Family& RAM_ = prometheus::BuildGauge() + .Name("RAM_usage_percent") + .Help("RAM usage percent by this process") + .Register(*registry_); + prometheus::Gauge& RAM_usage_percent_ = RAM_.Add({}); - //GPU Usage Percent - prometheus::Family &GPU_percent_ = prometheus::BuildGauge() - .Name("Gpu_usage_percent") - .Help("GPU_usage_percent ") - .Register(*registry_); + // GPU Usage Percent + prometheus::Family& GPU_percent_ = + prometheus::BuildGauge().Name("Gpu_usage_percent").Help("GPU_usage_percent ").Register(*registry_); - //GPU Mempry used - prometheus::Family &GPU_memory_usage_ = prometheus::BuildGauge() - .Name("GPU_memory_usage_total") - .Help("GPU memory usage total ") - .Register(*registry_); + // GPU Mempry used + prometheus::Family& GPU_memory_usage_ = + prometheus::BuildGauge().Name("GPU_memory_usage_total").Help("GPU memory usage total ").Register(*registry_); - prometheus::Family &query_index_type_per_second_ = prometheus::BuildGauge() - .Name("query_index_throughtout_per_microsecond") - .Help("query index throughtout per microsecond") - .Register(*registry_); - prometheus::Gauge - &query_index_IVF_type_per_second_gauge_ = query_index_type_per_second_.Add({{"IndexType", "IVF"}}); - prometheus::Gauge - &query_index_IDMAP_type_per_second_gauge_ = query_index_type_per_second_.Add({{"IndexType", "IDMAP"}}); + prometheus::Family& query_index_type_per_second_ = + prometheus::BuildGauge() + .Name("query_index_throughtout_per_microsecond") + .Help("query index throughtout per microsecond") + .Register(*registry_); + prometheus::Gauge& query_index_IVF_type_per_second_gauge_ = + query_index_type_per_second_.Add({{"IndexType", "IVF"}}); + prometheus::Gauge& query_index_IDMAP_type_per_second_gauge_ = + query_index_type_per_second_.Add({{"IndexType", "IDMAP"}}); - prometheus::Family &connection_ = prometheus::BuildGauge() - .Name("connection_number") - .Help("the number of connections") - .Register(*registry_); - prometheus::Gauge &connection_gauge_ = connection_.Add({}); + prometheus::Family& connection_ = + prometheus::BuildGauge().Name("connection_number").Help("the number of connections").Register(*registry_); + prometheus::Gauge& connection_gauge_ = connection_.Add({}); - prometheus::Family &keeping_alive_ = prometheus::BuildCounter() - .Name("keeping_alive_seconds_total") - .Help("total seconds of the serve alive") - .Register(*registry_); - prometheus::Counter &keeping_alive_counter_ = keeping_alive_.Add({}); + prometheus::Family& keeping_alive_ = prometheus::BuildCounter() + .Name("keeping_alive_seconds_total") + .Help("total seconds of the serve alive") + .Register(*registry_); + prometheus::Counter& keeping_alive_counter_ = keeping_alive_.Add({}); - prometheus::Family &octets_ = prometheus::BuildGauge() - .Name("octets_bytes_per_second") - .Help("octets bytes per second") - .Register(*registry_); - prometheus::Gauge &inoctets_gauge_ = octets_.Add({{"type", "inoctets"}}); - prometheus::Gauge &outoctets_gauge_ = octets_.Add({{"type", "outoctets"}}); + prometheus::Family& octets_ = + prometheus::BuildGauge().Name("octets_bytes_per_second").Help("octets bytes per second").Register(*registry_); + prometheus::Gauge& inoctets_gauge_ = octets_.Add({{"type", "inoctets"}}); + prometheus::Gauge& outoctets_gauge_ = octets_.Add({{"type", "outoctets"}}); - prometheus::Family &GPU_temperature_ = prometheus::BuildGauge() - .Name("GPU_temperature") - .Help("GPU temperature") - .Register(*registry_); + prometheus::Family& GPU_temperature_ = + prometheus::BuildGauge().Name("GPU_temperature").Help("GPU temperature").Register(*registry_); - prometheus::Family &CPU_temperature_ = prometheus::BuildGauge() - .Name("CPU_temperature") - .Help("CPU temperature") - .Register(*registry_); + prometheus::Family& CPU_temperature_ = + prometheus::BuildGauge().Name("CPU_temperature").Help("CPU temperature").Register(*registry_); }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/SystemInfo.cpp b/cpp/src/metrics/SystemInfo.cpp index 54d0c51943..68c1182726 100644 --- a/cpp/src/metrics/SystemInfo.cpp +++ b/cpp/src/metrics/SystemInfo.cpp @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. - #include "metrics/SystemInfo.h" +#include #include #include -#include #include -#include +#include #include #include @@ -37,7 +36,7 @@ SystemInfo::Init() { initialized_ = true; // initialize CPU information - FILE *file; + FILE* file; struct tms time_sample; char line[128]; last_cpu_ = times(&time_sample); @@ -54,7 +53,7 @@ SystemInfo::Init() { total_ram_ = GetPhysicalMemory(); fclose(file); - //initialize GPU information + // initialize GPU information nvmlReturn_t nvmlresult; nvmlresult = nvmlInit(); if (NVML_SUCCESS != nvmlresult) { @@ -67,7 +66,7 @@ SystemInfo::Init() { return; } - //initialize network traffic information + // initialize network traffic information std::pair in_and_out_octets = Octets(); in_octets_ = in_and_out_octets.first; out_octets_ = in_and_out_octets.second; @@ -75,10 +74,10 @@ SystemInfo::Init() { } uint64_t -SystemInfo::ParseLine(char *line) { +SystemInfo::ParseLine(char* line) { // This assumes that a digit will be found and the line ends in " Kb". int i = strlen(line); - const char *p = line; + const char* p = line; while (*p < '0' || *p > '9') p++; line[i - 3] = '\0'; i = atoi(p); @@ -90,15 +89,15 @@ SystemInfo::GetPhysicalMemory() { struct sysinfo memInfo; sysinfo(&memInfo); uint64_t totalPhysMem = memInfo.totalram; - //Multiply in next statement to avoid int overflow on right hand side... + // Multiply in next statement to avoid int overflow on right hand side... totalPhysMem *= memInfo.mem_unit; return totalPhysMem; } uint64_t SystemInfo::GetProcessUsedMemory() { - //Note: this value is in KB! - FILE *file = fopen("/proc/self/status", "r"); + // Note: this value is in KB! + FILE* file = fopen("/proc/self/status", "r"); constexpr uint64_t line_length = 128; uint64_t result = -1; constexpr uint64_t KB_SIZE = 1024; @@ -118,7 +117,7 @@ SystemInfo::GetProcessUsedMemory() { double SystemInfo::MemoryPercent() { if (!initialized_) Init(); - return (double) (GetProcessUsedMemory() * 100) / (double) total_ram_; + return (double)(GetProcessUsedMemory() * 100) / (double)total_ram_; } std::vector @@ -139,9 +138,9 @@ SystemInfo::CPUCorePercent() { } std::vector -SystemInfo::getTotalCpuTime(std::vector &work_time_array) { +SystemInfo::getTotalCpuTime(std::vector& work_time_array) { std::vector total_time_array; - FILE *file = fopen("/proc/stat", "r"); + FILE* file = fopen("/proc/stat", "r"); if (file == NULL) { perror("Could not open stat file"); return total_time_array; @@ -152,16 +151,15 @@ SystemInfo::getTotalCpuTime(std::vector &work_time_array) { for (int i = 0; i < num_processors_; i++) { char buffer[1024]; - char *ret = fgets(buffer, sizeof(buffer) - 1, file); + char* ret = fgets(buffer, sizeof(buffer) - 1, file); if (ret == NULL) { perror("Could not read stat file"); fclose(file); return total_time_array; } - sscanf(buffer, - "cpu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu", - &user, &nice, &system, &idle, &iowait, &irq, &softirq, &steal, &guest, &guestnice); + sscanf(buffer, "cpu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu", &user, &nice, &system, &idle, + &iowait, &irq, &softirq, &steal, &guest, &guestnice); work_time_array.push_back(user + nice + system); total_time_array.push_back(user + nice + system + idle + iowait + irq + softirq + steal); @@ -179,13 +177,11 @@ SystemInfo::CPUPercent() { double percent; now = times(&time_sample); - if (now <= last_cpu_ || time_sample.tms_stime < last_sys_cpu_ || - time_sample.tms_utime < last_user_cpu_) { - //Overflow detection. Just skip this value. + if (now <= last_cpu_ || time_sample.tms_stime < last_sys_cpu_ || time_sample.tms_utime < last_user_cpu_) { + // Overflow detection. Just skip this value. percent = -1.0; } else { - percent = (time_sample.tms_stime - last_sys_cpu_) + - (time_sample.tms_utime - last_user_cpu_); + percent = (time_sample.tms_stime - last_sys_cpu_) + (time_sample.tms_utime - last_user_cpu_); percent /= (now - last_cpu_); percent *= 100; } @@ -230,7 +226,7 @@ SystemInfo::CPUTemperature() { std::vector result; for (int i = 0; i <= num_physical_processors_; ++i) { std::string path = "/sys/class/thermal/thermal_zone" + std::to_string(i) + "/temp"; - FILE *file = fopen(path.data(), "r"); + FILE* file = fopen(path.data(), "r"); if (file == NULL) { perror("Could not open thermal file"); return result; @@ -261,7 +257,7 @@ SystemInfo::GPUMemoryUsed() { std::pair SystemInfo::Octets() { pid_t pid = getpid(); -// const std::string filename = "/proc/"+std::to_string(pid)+"/net/netstat"; + // const std::string filename = "/proc/"+std::to_string(pid)+"/net/netstat"; const std::string filename = "/proc/net/netstat"; std::ifstream file(filename); std::string lastline = ""; @@ -293,6 +289,6 @@ SystemInfo::Octets() { return res; } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/SystemInfo.h b/cpp/src/metrics/SystemInfo.h index 802cbb0cce..7fda0730e6 100644 --- a/cpp/src/metrics/SystemInfo.h +++ b/cpp/src/metrics/SystemInfo.h @@ -15,20 +15,19 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include +#include +#include +#include #include #include +#include #include -#include -#include -#include #include #include -#include #include +#include namespace zilliz { namespace milvus { @@ -43,72 +42,94 @@ class SystemInfo { std::chrono::system_clock::time_point net_time_ = std::chrono::system_clock::now(); int num_processors_ = 0; int num_physical_processors_ = 0; - //number of GPU + // number of GPU uint32_t num_device_ = 0; uint64_t in_octets_ = 0; uint64_t out_octets_ = 0; bool initialized_ = false; public: - static SystemInfo & + static SystemInfo& GetInstance() { static SystemInfo instance; return instance; } - void Init(); + void + Init(); - int num_processor() const { + int + num_processor() const { return num_processors_; } - int num_physical_processors() const { + int + num_physical_processors() const { return num_physical_processors_; } - uint32_t num_device() const { + uint32_t + num_device() const { return num_device_; } - uint64_t get_inoctets() { + uint64_t + get_inoctets() { return in_octets_; } - uint64_t get_octets() { + uint64_t + get_octets() { return out_octets_; } - std::chrono::system_clock::time_point get_nettime() { + std::chrono::system_clock::time_point + get_nettime() { return net_time_; } - void set_inoctets(uint64_t value) { + void + set_inoctets(uint64_t value) { in_octets_ = value; } - void set_outoctets(uint64_t value) { + void + set_outoctets(uint64_t value) { out_octets_ = value; } - void set_nettime() { + void + set_nettime() { net_time_ = std::chrono::system_clock::now(); } - uint64_t ParseLine(char *line); - uint64_t GetPhysicalMemory(); - uint64_t GetProcessUsedMemory(); - double MemoryPercent(); - double CPUPercent(); - std::pair Octets(); - std::vector GPUMemoryTotal(); - std::vector GPUMemoryUsed(); + uint64_t + ParseLine(char* line); + uint64_t + GetPhysicalMemory(); + uint64_t + GetProcessUsedMemory(); + double + MemoryPercent(); + double + CPUPercent(); + std::pair + Octets(); + std::vector + GPUMemoryTotal(); + std::vector + GPUMemoryUsed(); - std::vector CPUCorePercent(); - std::vector getTotalCpuTime(std::vector &workTime); - std::vector GPUTemperature(); - std::vector CPUTemperature(); + std::vector + CPUCorePercent(); + std::vector + getTotalCpuTime(std::vector& workTime); + std::vector + GPUTemperature(); + std::vector + CPUTemperature(); }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Algorithm.cpp b/cpp/src/scheduler/Algorithm.cpp index 10a585304a..48fee94d9c 100644 --- a/cpp/src/scheduler/Algorithm.cpp +++ b/cpp/src/scheduler/Algorithm.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/Algorithm.h" #include @@ -29,10 +28,8 @@ namespace scheduler { constexpr uint64_t MAXINT = std::numeric_limits::max(); uint64_t -ShortestPath(const ResourcePtr &src, - const ResourcePtr &dest, - const ResourceMgrPtr &res_mgr, - std::vector &path) { +ShortestPath(const ResourcePtr& src, const ResourcePtr& dest, const ResourceMgrPtr& res_mgr, + std::vector& path) { std::vector> paths; uint64_t num_of_resources = res_mgr->GetAllResources().size(); @@ -43,7 +40,7 @@ ShortestPath(const ResourcePtr &src, name_id_map.insert(std::make_pair(res_mgr->GetAllResources().at(i)->name(), i)); } - std::vector > dis_matrix; + std::vector> dis_matrix; dis_matrix.resize(num_of_resources); for (uint64_t i = 0; i < num_of_resources; ++i) { dis_matrix[i].resize(num_of_resources); @@ -55,11 +52,11 @@ ShortestPath(const ResourcePtr &src, std::vector vis(num_of_resources, false); std::vector dis(num_of_resources, MAXINT); - for (auto &res : res_mgr->GetAllResources()) { + for (auto& res : res_mgr->GetAllResources()) { auto cur_node = std::static_pointer_cast(res); auto cur_neighbours = cur_node->GetNeighbours(); - for (auto &neighbour : cur_neighbours) { + for (auto& neighbour : cur_neighbours) { auto neighbour_res = std::static_pointer_cast(neighbour.neighbour_node.lock()); dis_matrix[name_id_map.at(res->name())][name_id_map.at(neighbour_res->name())] = neighbour.connection.transport_cost(); @@ -107,6 +104,6 @@ ShortestPath(const ResourcePtr &src, return dis[name_id_map.at(dest->name())]; } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Algorithm.h b/cpp/src/scheduler/Algorithm.h index d7e0233ba0..ed0f17d762 100644 --- a/cpp/src/scheduler/Algorithm.h +++ b/cpp/src/scheduler/Algorithm.h @@ -15,23 +15,20 @@ // specific language governing permissions and limitations // under the License. - -#include "resource/Resource.h" #include "ResourceMgr.h" +#include "resource/Resource.h" -#include #include +#include namespace zilliz { namespace milvus { namespace scheduler { uint64_t -ShortestPath(const ResourcePtr &src, - const ResourcePtr &dest, - const ResourceMgrPtr &res_mgr, - std::vector &path); +ShortestPath(const ResourcePtr& src, const ResourcePtr& dest, const ResourceMgrPtr& res_mgr, + std::vector& path); -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Definition.h b/cpp/src/scheduler/Definition.h index ce41aca48d..c886aafdac 100644 --- a/cpp/src/scheduler/Definition.h +++ b/cpp/src/scheduler/Definition.h @@ -15,20 +15,20 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include -#include "db/meta/MetaTypes.h" #include "db/engine/EngineFactory.h" #include "db/engine/ExecutionEngine.h" +#include "db/meta/MetaTypes.h" namespace zilliz { namespace milvus { @@ -42,6 +42,6 @@ using EngineFactory = engine::EngineFactory; using EngineType = engine::EngineType; using MetricType = engine::MetricType; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/JobMgr.cpp b/cpp/src/scheduler/JobMgr.cpp index 0406e98c49..92cf1189fd 100644 --- a/cpp/src/scheduler/JobMgr.cpp +++ b/cpp/src/scheduler/JobMgr.cpp @@ -16,8 +16,8 @@ // under the License. #include "scheduler/JobMgr.h" -#include "task/Task.h" #include "TaskCreator.h" +#include "task/Task.h" #include @@ -25,8 +25,7 @@ namespace zilliz { namespace milvus { namespace scheduler { -JobMgr::JobMgr(ResourceMgrPtr res_mgr) - : res_mgr_(std::move(res_mgr)) { +JobMgr::JobMgr(ResourceMgrPtr res_mgr) : res_mgr_(std::move(res_mgr)) { } void @@ -47,7 +46,7 @@ JobMgr::Stop() { } void -JobMgr::Put(const JobPtr &job) { +JobMgr::Put(const JobPtr& job) { { std::lock_guard lock(mutex_); queue_.push(job); @@ -59,9 +58,7 @@ void JobMgr::worker_function() { while (running_) { std::unique_lock lock(mutex_); - cv_.wait(lock, [this] { - return !queue_.empty(); - }); + cv_.wait(lock, [this] { return !queue_.empty(); }); auto job = queue_.front(); queue_.pop(); lock.unlock(); @@ -73,7 +70,7 @@ JobMgr::worker_function() { auto disk_list = res_mgr_->GetDiskResources(); if (!disk_list.empty()) { if (auto disk = disk_list[0].lock()) { - for (auto &task : tasks) { + for (auto& task : tasks) { disk->task_table().Put(task); } } @@ -82,10 +79,10 @@ JobMgr::worker_function() { } std::vector -JobMgr::build_task(const JobPtr &job) { +JobMgr::build_task(const JobPtr& job) { return TaskCreator::Create(job); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/JobMgr.h b/cpp/src/scheduler/JobMgr.h index 49ba9154e3..66adff18d2 100644 --- a/cpp/src/scheduler/JobMgr.h +++ b/cpp/src/scheduler/JobMgr.h @@ -16,20 +16,20 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include "ResourceMgr.h" #include "job/Job.h" #include "task/Task.h" -#include "ResourceMgr.h" namespace zilliz { namespace milvus { @@ -47,14 +47,14 @@ class JobMgr { public: void - Put(const JobPtr &job); + Put(const JobPtr& job); private: void worker_function(); std::vector - build_task(const JobPtr &job); + build_task(const JobPtr& job); private: bool running_ = false; @@ -70,6 +70,6 @@ class JobMgr { using JobMgrPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceFactory.cpp b/cpp/src/scheduler/ResourceFactory.cpp index de9b5bc717..f89e8207e4 100644 --- a/cpp/src/scheduler/ResourceFactory.cpp +++ b/cpp/src/scheduler/ResourceFactory.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/ResourceFactory.h" namespace zilliz { @@ -23,10 +22,7 @@ namespace milvus { namespace scheduler { std::shared_ptr -ResourceFactory::Create(const std::string &name, - const std::string &type, - uint64_t device_id, - bool enable_loader, +ResourceFactory::Create(const std::string& name, const std::string& type, uint64_t device_id, bool enable_loader, bool enable_executor) { if (type == "DISK") { return std::make_shared(name, device_id, enable_loader, enable_executor); @@ -39,6 +35,6 @@ ResourceFactory::Create(const std::string &name, } } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceFactory.h b/cpp/src/scheduler/ResourceFactory.h index f7a47ef1e5..bf663a7db8 100644 --- a/cpp/src/scheduler/ResourceFactory.h +++ b/cpp/src/scheduler/ResourceFactory.h @@ -17,13 +17,13 @@ #pragma once -#include #include +#include -#include "resource/Resource.h" #include "resource/CpuResource.h" -#include "resource/GpuResource.h" #include "resource/DiskResource.h" +#include "resource/GpuResource.h" +#include "resource/Resource.h" namespace zilliz { namespace milvus { @@ -32,13 +32,10 @@ namespace scheduler { class ResourceFactory { public: static std::shared_ptr - Create(const std::string &name, - const std::string &type, - uint64_t device_id, - bool enable_loader = true, + Create(const std::string& name, const std::string& type, uint64_t device_id, bool enable_loader = true, bool enable_executor = true); }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceMgr.cpp b/cpp/src/scheduler/ResourceMgr.cpp index 6067b2eb01..eefa30560c 100644 --- a/cpp/src/scheduler/ResourceMgr.cpp +++ b/cpp/src/scheduler/ResourceMgr.cpp @@ -26,7 +26,7 @@ namespace scheduler { void ResourceMgr::Start() { std::lock_guard lck(resources_mutex_); - for (auto &resource : resources_) { + for (auto& resource : resources_) { resource->Start(); } running_ = true; @@ -44,13 +44,13 @@ ResourceMgr::Stop() { worker_thread_.join(); std::lock_guard lck(resources_mutex_); - for (auto &resource : resources_) { + for (auto& resource : resources_) { resource->Stop(); } } ResourceWPtr -ResourceMgr::Add(ResourcePtr &&resource) { +ResourceMgr::Add(ResourcePtr&& resource) { ResourceWPtr ret(resource); std::lock_guard lck(resources_mutex_); @@ -70,13 +70,13 @@ ResourceMgr::Add(ResourcePtr &&resource) { } bool -ResourceMgr::Connect(const std::string &name1, const std::string &name2, Connection &connection) { +ResourceMgr::Connect(const std::string& name1, const std::string& name2, Connection& connection) { auto res1 = GetResource(name1); auto res2 = GetResource(name2); if (res1 && res2) { res1->AddNeighbour(std::static_pointer_cast(res2), connection); // TODO: enable when task balance supported -// res2->AddNeighbour(std::static_pointer_cast(res1), connection); + // res2->AddNeighbour(std::static_pointer_cast(res1), connection); return true; } return false; @@ -92,7 +92,7 @@ ResourceMgr::Clear() { std::vector ResourceMgr::GetComputeResources() { std::vector result; - for (auto &resource : resources_) { + for (auto& resource : resources_) { if (resource->HasExecutor()) { result.emplace_back(resource); } @@ -102,7 +102,7 @@ ResourceMgr::GetComputeResources() { ResourcePtr ResourceMgr::GetResource(ResourceType type, uint64_t device_id) { - for (auto &resource : resources_) { + for (auto& resource : resources_) { if (resource->type() == type && resource->device_id() == device_id) { return resource; } @@ -111,8 +111,8 @@ ResourceMgr::GetResource(ResourceType type, uint64_t device_id) { } ResourcePtr -ResourceMgr::GetResource(const std::string &name) { - for (auto &resource : resources_) { +ResourceMgr::GetResource(const std::string& name) { + for (auto& resource : resources_) { if (resource->name() == name) { return resource; } @@ -128,7 +128,7 @@ ResourceMgr::GetNumOfResource() const { uint64_t ResourceMgr::GetNumOfComputeResource() const { uint64_t count = 0; - for (auto &res : resources_) { + for (auto& res : resources_) { if (res->HasExecutor()) { ++count; } @@ -139,7 +139,7 @@ ResourceMgr::GetNumOfComputeResource() const { uint64_t ResourceMgr::GetNumGpuResource() const { uint64_t num = 0; - for (auto &res : resources_) { + for (auto& res : resources_) { if (res->type() == ResourceType::GPU) { num++; } @@ -153,7 +153,7 @@ ResourceMgr::Dump() { for (uint64_t i = 0; i < resources_.size(); ++i) { str += "Resource No." + std::to_string(i) + ":\n"; - //str += resources_[i]->Dump(); + // str += resources_[i]->Dump(); } return str; @@ -163,7 +163,7 @@ std::string ResourceMgr::DumpTaskTables() { std::stringstream ss; ss << ">>>>>>>>>>>>>>>ResourceMgr::DumpTaskTable<<<<<<<<<<<<<<<" << std::endl; - for (auto &resource : resources_) { + for (auto& resource : resources_) { ss << resource->Dump() << std::endl; ss << resource->task_table().Dump(); ss << resource->Dump() << std::endl << std::endl; @@ -172,7 +172,7 @@ ResourceMgr::DumpTaskTables() { } void -ResourceMgr::post_event(const EventPtr &event) { +ResourceMgr::post_event(const EventPtr& event) { { std::lock_guard lock(event_mutex_); queue_.emplace(event); @@ -184,9 +184,7 @@ void ResourceMgr::event_process() { while (running_) { std::unique_lock lock(event_mutex_); - event_cv_.wait(lock, [this] { - return !queue_.empty(); - }); + event_cv_.wait(lock, [this] { return !queue_.empty(); }); auto event = queue_.front(); queue_.pop(); @@ -201,6 +199,6 @@ ResourceMgr::event_process() { } } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceMgr.h b/cpp/src/scheduler/ResourceMgr.h index d03408a7df..b6720a2054 100644 --- a/cpp/src/scheduler/ResourceMgr.h +++ b/cpp/src/scheduler/ResourceMgr.h @@ -17,13 +17,13 @@ #pragma once -#include -#include +#include #include #include #include +#include #include -#include +#include #include "resource/Resource.h" #include "utils/Log.h" @@ -45,10 +45,10 @@ class ResourceMgr { Stop(); ResourceWPtr - Add(ResourcePtr &&resource); + Add(ResourcePtr&& resource); bool - Connect(const std::string &res1, const std::string &res2, Connection &connection); + Connect(const std::string& res1, const std::string& res2, Connection& connection); void Clear(); @@ -60,7 +60,7 @@ class ResourceMgr { public: /******** Management Interface ********/ - inline std::vector & + inline std::vector& GetDiskResources() { return disk_resources_; } @@ -78,7 +78,7 @@ class ResourceMgr { GetResource(ResourceType type, uint64_t device_id); ResourcePtr - GetResource(const std::string &name); + GetResource(const std::string& name); uint64_t GetNumOfResource() const; @@ -102,7 +102,7 @@ class ResourceMgr { private: void - post_event(const EventPtr &event); + post_event(const EventPtr& event); void event_process(); @@ -125,6 +125,6 @@ class ResourceMgr { using ResourceMgrPtr = std::shared_ptr; using ResourceMgrWPtr = std::weak_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 71b40de9ee..5c04eeb931 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -15,17 +15,16 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/SchedInst.h" -#include "server/Config.h" #include "ResourceFactory.h" -#include "knowhere/index/vector_index/IndexGPUIVF.h" #include "Utils.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "server/Config.h" -#include #include -#include #include +#include +#include namespace zilliz { namespace milvus { @@ -42,7 +41,7 @@ std::mutex JobMgrInst::mutex_; void load_simple_config() { - server::Config &config = server::Config::GetInstance(); + server::Config& config = server::Config::GetInstance(); std::string mode; config.GetResourceConfigMode(mode); std::vector pool; @@ -50,7 +49,7 @@ load_simple_config() { bool cpu = false; std::set gpu_ids; - for (auto &resource : pool) { + for (auto& resource : pool) { if (resource == "cpu") { cpu = true; break; @@ -78,7 +77,7 @@ load_simple_config() { ResMgrInst::GetInstance()->Connect("disk", "cpu", io); auto pcie = Connection("pcie", 12000); - for (auto &gpu_id : gpu_ids) { + for (auto& gpu_id : gpu_ids) { ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), io); } @@ -87,77 +86,77 @@ load_simple_config() { void load_advance_config() { -// try { -// server::ConfigNode &config = server::Config::GetInstance().GetConfig(server::CONFIG_RESOURCE); -// -// if (config.GetChildren().empty()) throw "resource_config null exception"; -// -// auto resources = config.GetChild(server::CONFIG_RESOURCES).GetChildren(); -// -// if (resources.empty()) throw "Children of resource_config null exception"; -// -// for (auto &resource : resources) { -// auto &resname = resource.first; -// auto &resconf = resource.second; -// auto type = resconf.GetValue(server::CONFIG_RESOURCE_TYPE); -//// auto memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_MEMORY); -// auto device_id = resconf.GetInt64Value(server::CONFIG_RESOURCE_DEVICE_ID); -//// auto enable_loader = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_LOADER); -// auto enable_loader = true; -// auto enable_executor = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_EXECUTOR); -// auto pinned_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_PIN_MEMORY); -// auto temp_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_TEMP_MEMORY); -// auto resource_num = resconf.GetInt64Value(server::CONFIG_RESOURCE_NUM); -// -// auto res = ResMgrInst::GetInstance()->Add(ResourceFactory::Create(resname, -// type, -// device_id, -// enable_loader, -// enable_executor)); -// -// if (res.lock()->type() == ResourceType::GPU) { -// auto pinned_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_PIN_MEMORY, 300); -// auto temp_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_TEMP_MEMORY, 300); -// auto resource_num = resconf.GetInt64Value(server::CONFIG_RESOURCE_NUM, 2); -// pinned_memory = 1024 * 1024 * pinned_memory; -// temp_memory = 1024 * 1024 * temp_memory; -// knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, -// pinned_memory, -// temp_memory, -// resource_num); -// } -// } -// -// knowhere::FaissGpuResourceMgr::GetInstance().InitResource(); -// -// auto connections = config.GetChild(server::CONFIG_RESOURCE_CONNECTIONS).GetChildren(); -// if (connections.empty()) throw "connections config null exception"; -// for (auto &conn : connections) { -// auto &connect_name = conn.first; -// auto &connect_conf = conn.second; -// auto connect_speed = connect_conf.GetInt64Value(server::CONFIG_SPEED_CONNECTIONS); -// auto connect_endpoint = connect_conf.GetValue(server::CONFIG_ENDPOINT_CONNECTIONS); -// -// std::string delimiter = "==="; -// std::string left = connect_endpoint.substr(0, connect_endpoint.find(delimiter)); -// std::string right = connect_endpoint.substr(connect_endpoint.find(delimiter) + 3, -// connect_endpoint.length()); -// -// auto connection = Connection(connect_name, connect_speed); -// ResMgrInst::GetInstance()->Connect(left, right, connection); -// } -// } catch (const char *msg) { -// SERVER_LOG_ERROR << msg; -// // TODO: throw exception instead -// exit(-1); -//// throw std::exception(); -// } + // try { + // server::ConfigNode &config = server::Config::GetInstance().GetConfig(server::CONFIG_RESOURCE); + // + // if (config.GetChildren().empty()) throw "resource_config null exception"; + // + // auto resources = config.GetChild(server::CONFIG_RESOURCES).GetChildren(); + // + // if (resources.empty()) throw "Children of resource_config null exception"; + // + // for (auto &resource : resources) { + // auto &resname = resource.first; + // auto &resconf = resource.second; + // auto type = resconf.GetValue(server::CONFIG_RESOURCE_TYPE); + //// auto memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_MEMORY); + // auto device_id = resconf.GetInt64Value(server::CONFIG_RESOURCE_DEVICE_ID); + //// auto enable_loader = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_LOADER); + // auto enable_loader = true; + // auto enable_executor = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_EXECUTOR); + // auto pinned_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_PIN_MEMORY); + // auto temp_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_TEMP_MEMORY); + // auto resource_num = resconf.GetInt64Value(server::CONFIG_RESOURCE_NUM); + // + // auto res = ResMgrInst::GetInstance()->Add(ResourceFactory::Create(resname, + // type, + // device_id, + // enable_loader, + // enable_executor)); + // + // if (res.lock()->type() == ResourceType::GPU) { + // auto pinned_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_PIN_MEMORY, 300); + // auto temp_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_TEMP_MEMORY, 300); + // auto resource_num = resconf.GetInt64Value(server::CONFIG_RESOURCE_NUM, 2); + // pinned_memory = 1024 * 1024 * pinned_memory; + // temp_memory = 1024 * 1024 * temp_memory; + // knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, + // pinned_memory, + // temp_memory, + // resource_num); + // } + // } + // + // knowhere::FaissGpuResourceMgr::GetInstance().InitResource(); + // + // auto connections = config.GetChild(server::CONFIG_RESOURCE_CONNECTIONS).GetChildren(); + // if (connections.empty()) throw "connections config null exception"; + // for (auto &conn : connections) { + // auto &connect_name = conn.first; + // auto &connect_conf = conn.second; + // auto connect_speed = connect_conf.GetInt64Value(server::CONFIG_SPEED_CONNECTIONS); + // auto connect_endpoint = connect_conf.GetValue(server::CONFIG_ENDPOINT_CONNECTIONS); + // + // std::string delimiter = "==="; + // std::string left = connect_endpoint.substr(0, connect_endpoint.find(delimiter)); + // std::string right = connect_endpoint.substr(connect_endpoint.find(delimiter) + 3, + // connect_endpoint.length()); + // + // auto connection = Connection(connect_name, connect_speed); + // ResMgrInst::GetInstance()->Connect(left, right, connection); + // } + // } catch (const char *msg) { + // SERVER_LOG_ERROR << msg; + // // TODO: throw exception instead + // exit(-1); + //// throw std::exception(); + // } } void StartSchedulerService() { load_simple_config(); -// load_advance_config(); + // load_advance_config(); ResMgrInst::GetInstance()->Start(); SchedInst::GetInstance()->Start(); JobMgrInst::GetInstance()->Start(); @@ -170,6 +169,6 @@ StopSchedulerService() { ResMgrInst::GetInstance()->Stop(); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/SchedInst.h b/cpp/src/scheduler/SchedInst.h index 4cca6ec5a9..9947e9f62b 100644 --- a/cpp/src/scheduler/SchedInst.h +++ b/cpp/src/scheduler/SchedInst.h @@ -17,12 +17,12 @@ #pragma once +#include "JobMgr.h" #include "ResourceMgr.h" #include "Scheduler.h" -#include "JobMgr.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -88,6 +88,6 @@ StartSchedulerService(); void StopSchedulerService(); -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Scheduler.cpp b/cpp/src/scheduler/Scheduler.cpp index 24f7bfe73b..5cffa99968 100644 --- a/cpp/src/scheduler/Scheduler.cpp +++ b/cpp/src/scheduler/Scheduler.cpp @@ -16,10 +16,10 @@ // under the License. #include "scheduler/Scheduler.h" +#include "Algorithm.h" +#include "action/Action.h" #include "cache/GpuCacheMgr.h" #include "event/LoadCompletedEvent.h" -#include "action/Action.h" -#include "Algorithm.h" #include @@ -27,9 +27,7 @@ namespace zilliz { namespace milvus { namespace scheduler { -Scheduler::Scheduler(ResourceMgrWPtr res_mgr) - : running_(false), - res_mgr_(std::move(res_mgr)) { +Scheduler::Scheduler(ResourceMgrWPtr res_mgr) : running_(false), res_mgr_(std::move(res_mgr)) { if (auto mgr = res_mgr_.lock()) { mgr->RegisterSubscriber(std::bind(&Scheduler::PostEvent, this, std::placeholders::_1)); } @@ -61,7 +59,7 @@ Scheduler::Stop() { } void -Scheduler::PostEvent(const EventPtr &event) { +Scheduler::PostEvent(const EventPtr& event) { { std::lock_guard lock(event_mutex_); event_queue_.push(event); @@ -78,9 +76,7 @@ void Scheduler::worker_function() { while (running_) { std::unique_lock lock(event_mutex_); - event_cv_.wait(lock, [this] { - return !event_queue_.empty(); - }); + event_cv_.wait(lock, [this] { return !event_queue_.empty(); }); auto event = event_queue_.front(); event_queue_.pop(); if (event == nullptr) { @@ -92,14 +88,14 @@ Scheduler::worker_function() { } void -Scheduler::Process(const EventPtr &event) { +Scheduler::Process(const EventPtr& event) { auto process_event = event_register_.at(static_cast(event->Type())); process_event(event); } // TODO: refactor the function void -Scheduler::OnLoadCompleted(const EventPtr &event) { +Scheduler::OnLoadCompleted(const EventPtr& event) { auto load_completed_event = std::static_pointer_cast(event); if (auto resource = event->resource_.lock()) { resource->WakeupExecutor(); @@ -118,31 +114,29 @@ Scheduler::OnLoadCompleted(const EventPtr &event) { Action::PushTaskToAllNeighbour(load_completed_event->task_table_item_->task, resource); break; } - default: { - break; - } + default: { break; } } } } void -Scheduler::OnStartUp(const EventPtr &event) { +Scheduler::OnStartUp(const EventPtr& event) { if (auto resource = event->resource_.lock()) { resource->WakeupLoader(); } } void -Scheduler::OnFinishTask(const EventPtr &event) { +Scheduler::OnFinishTask(const EventPtr& event) { } void -Scheduler::OnTaskTableUpdated(const EventPtr &event) { +Scheduler::OnTaskTableUpdated(const EventPtr& event) { if (auto resource = event->resource_.lock()) { resource->WakeupLoader(); } } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Scheduler.h b/cpp/src/scheduler/Scheduler.h index 073f7eeb0d..e72e40d1ba 100644 --- a/cpp/src/scheduler/Scheduler.h +++ b/cpp/src/scheduler/Scheduler.h @@ -18,14 +18,14 @@ #pragma once #include -#include #include -#include #include +#include +#include #include -#include "resource/Resource.h" #include "ResourceMgr.h" +#include "resource/Resource.h" #include "utils/Log.h" namespace zilliz { @@ -37,8 +37,8 @@ class Scheduler { public: explicit Scheduler(ResourceMgrWPtr res_mgr); - Scheduler(const Scheduler &) = delete; - Scheduler(Scheduler &&) = delete; + Scheduler(const Scheduler&) = delete; + Scheduler(Scheduler&&) = delete; /* * Start worker thread; @@ -56,7 +56,7 @@ class Scheduler { * Post event to scheduler event queue; */ void - PostEvent(const EventPtr &event); + PostEvent(const EventPtr& event); /* * Dump as string; @@ -74,7 +74,7 @@ class Scheduler { * Pull task from neighbours; */ void - OnStartUp(const EventPtr &event); + OnStartUp(const EventPtr& event); /* * Process finish task events; @@ -83,7 +83,7 @@ class Scheduler { * Pull task from neighbours; */ void - OnFinishTask(const EventPtr &event); + OnFinishTask(const EventPtr& event); /* * Process copy completed events; @@ -93,7 +93,7 @@ class Scheduler { * Pull task from neighbours; */ void - OnLoadCompleted(const EventPtr &event); + OnLoadCompleted(const EventPtr& event); /* * Process task table updated events, which happened on task_table->put; @@ -102,14 +102,14 @@ class Scheduler { * Push task to neighbours; */ void - OnTaskTableUpdated(const EventPtr &event); + OnTaskTableUpdated(const EventPtr& event); private: /* * Dispatch event to event handler; */ void - Process(const EventPtr &event); + Process(const EventPtr& event); /* * Called by worker_thread_; @@ -131,6 +131,6 @@ class Scheduler { using SchedulerPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/TaskCreator.cpp b/cpp/src/scheduler/TaskCreator.cpp index 8b0378c646..4f8979703f 100644 --- a/cpp/src/scheduler/TaskCreator.cpp +++ b/cpp/src/scheduler/TaskCreator.cpp @@ -24,7 +24,7 @@ namespace milvus { namespace scheduler { std::vector -TaskCreator::Create(const JobPtr &job) { +TaskCreator::Create(const JobPtr& job) { switch (job->type()) { case JobType::SEARCH: { return Create(std::static_pointer_cast(job)); @@ -40,9 +40,9 @@ TaskCreator::Create(const JobPtr &job) { } std::vector -TaskCreator::Create(const SearchJobPtr &job) { +TaskCreator::Create(const SearchJobPtr& job) { std::vector tasks; - for (auto &index_file : job->index_files()) { + for (auto& index_file : job->index_files()) { auto task = std::make_shared(index_file.second); task->label() = std::make_shared(); task->job_ = job; @@ -53,7 +53,7 @@ TaskCreator::Create(const SearchJobPtr &job) { } std::vector -TaskCreator::Create(const DeleteJobPtr &job) { +TaskCreator::Create(const DeleteJobPtr& job) { std::vector tasks; auto task = std::make_shared(job); task->label() = std::make_shared(); @@ -63,6 +63,6 @@ TaskCreator::Create(const DeleteJobPtr &job) { return tasks; } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/TaskCreator.h b/cpp/src/scheduler/TaskCreator.h index 81cb25010f..5ae6a0763a 100644 --- a/cpp/src/scheduler/TaskCreator.h +++ b/cpp/src/scheduler/TaskCreator.h @@ -16,23 +16,23 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include "job/DeleteJob.h" #include "job/Job.h" #include "job/SearchJob.h" -#include "job/DeleteJob.h" -#include "task/Task.h" -#include "task/SearchTask.h" #include "task/DeleteTask.h" +#include "task/SearchTask.h" +#include "task/Task.h" namespace zilliz { namespace milvus { @@ -41,16 +41,16 @@ namespace scheduler { class TaskCreator { public: static std::vector - Create(const JobPtr &job); + Create(const JobPtr& job); public: static std::vector - Create(const SearchJobPtr &job); + Create(const SearchJobPtr& job); static std::vector - Create(const DeleteJobPtr &job); + Create(const DeleteJobPtr& job); }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/TaskTable.cpp b/cpp/src/scheduler/TaskTable.cpp index a7343ee509..db06abd966 100644 --- a/cpp/src/scheduler/TaskTable.cpp +++ b/cpp/src/scheduler/TaskTable.cpp @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/TaskTable.h" -#include "event/TaskTableUpdatedEvent.h" #include "Utils.h" +#include "event/TaskTableUpdatedEvent.h" -#include -#include #include +#include +#include namespace zilliz { namespace milvus { @@ -31,20 +30,29 @@ namespace scheduler { std::string ToString(TaskTableItemState state) { switch (state) { - case TaskTableItemState::INVALID: return "INVALID"; - case TaskTableItemState::START: return "START"; - case TaskTableItemState::LOADING: return "LOADING"; - case TaskTableItemState::LOADED: return "LOADED"; - case TaskTableItemState::EXECUTING: return "EXECUTING"; - case TaskTableItemState::EXECUTED: return "EXECUTED"; - case TaskTableItemState::MOVING: return "MOVING"; - case TaskTableItemState::MOVED: return "MOVED"; - default: return ""; + case TaskTableItemState::INVALID: + return "INVALID"; + case TaskTableItemState::START: + return "START"; + case TaskTableItemState::LOADING: + return "LOADING"; + case TaskTableItemState::LOADED: + return "LOADED"; + case TaskTableItemState::EXECUTING: + return "EXECUTING"; + case TaskTableItemState::EXECUTED: + return "EXECUTED"; + case TaskTableItemState::MOVING: + return "MOVING"; + case TaskTableItemState::MOVED: + return "MOVED"; + default: + return ""; } } std::string -ToString(const TaskTimestamp ×tamp) { +ToString(const TaskTimestamp& timestamp) { std::stringstream ss; ss << " subscriber) { @@ -117,7 +117,7 @@ class TaskTable { * Called by DBImpl; */ void - Put(std::vector &tasks); + Put(std::vector& tasks); /* * Return task table item reference; @@ -130,8 +130,8 @@ class TaskTable { * Remove sequence task which is DONE or MOVED from front; * Called by ? */ -// void -// Clear(); + // void + // Clear(); /* * Return true if task table empty, otherwise false; @@ -150,16 +150,17 @@ class TaskTable { } public: - TaskTableItemPtr & - operator[](uint64_t index) { + TaskTableItemPtr& operator[](uint64_t index) { return table_[index]; } - std::deque::iterator begin() { + std::deque::iterator + begin() { return table_.begin(); } - std::deque::iterator end() { + std::deque::iterator + end() { return table_.end(); } @@ -254,6 +255,6 @@ class TaskTable { uint64_t last_finish_ = -1; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Utils.cpp b/cpp/src/scheduler/Utils.cpp index bb19950ffe..b50fe4adb9 100644 --- a/cpp/src/scheduler/Utils.cpp +++ b/cpp/src/scheduler/Utils.cpp @@ -15,11 +15,10 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/Utils.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -40,6 +39,6 @@ get_num_gpu() { return n_devices; } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Utils.h b/cpp/src/scheduler/Utils.h index c69028f0fa..eec5f8ba68 100644 --- a/cpp/src/scheduler/Utils.h +++ b/cpp/src/scheduler/Utils.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include namespace zilliz { @@ -28,6 +27,6 @@ get_current_timestamp(); uint64_t get_num_gpu(); -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/action/Action.h b/cpp/src/scheduler/action/Action.h index a5f67aa98d..b75dc4515f 100644 --- a/cpp/src/scheduler/action/Action.h +++ b/cpp/src/scheduler/action/Action.h @@ -17,8 +17,8 @@ #pragma once -#include "scheduler/resource/Resource.h" #include "scheduler/ResourceMgr.h" +#include "scheduler/resource/Resource.h" #include @@ -29,23 +29,22 @@ namespace scheduler { class Action { public: static void - PushTaskToNeighbourRandomly(const TaskPtr &task, const ResourcePtr &self); + PushTaskToNeighbourRandomly(const TaskPtr& task, const ResourcePtr& self); static void - PushTaskToAllNeighbour(const TaskPtr &task, const ResourcePtr &self); + PushTaskToAllNeighbour(const TaskPtr& task, const ResourcePtr& self); static void - PushTaskToResource(const TaskPtr &task, const ResourcePtr &dest); + PushTaskToResource(const TaskPtr& task, const ResourcePtr& dest); static void DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr resource, std::shared_ptr event); static void - SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, - ResourcePtr resource, + SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr resource, std::shared_ptr event); }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp index 909112eb62..c7514b1e2d 100644 --- a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp @@ -15,26 +15,25 @@ // specific language governing permissions and limitations // under the License. - #include #include #include "../Algorithm.h" -#include "src/cache/GpuCacheMgr.h" #include "Action.h" +#include "src/cache/GpuCacheMgr.h" namespace zilliz { namespace milvus { namespace scheduler { std::vector -get_neighbours(const ResourcePtr &self) { +get_neighbours(const ResourcePtr& self) { std::vector neighbours; - for (auto &neighbour_node : self->GetNeighbours()) { + for (auto& neighbour_node : self->GetNeighbours()) { auto node = neighbour_node.neighbour_node.lock(); if (not node) continue; auto resource = std::static_pointer_cast(node); -// if (not resource->HasExecutor()) continue; + // if (not resource->HasExecutor()) continue; neighbours.emplace_back(resource); } @@ -42,14 +41,14 @@ get_neighbours(const ResourcePtr &self) { } std::vector> -get_neighbours_with_connetion(const ResourcePtr &self) { +get_neighbours_with_connetion(const ResourcePtr& self) { std::vector> neighbours; - for (auto &neighbour_node : self->GetNeighbours()) { + for (auto& neighbour_node : self->GetNeighbours()) { auto node = neighbour_node.neighbour_node.lock(); if (not node) continue; auto resource = std::static_pointer_cast(node); -// if (not resource->HasExecutor()) continue; + // if (not resource->HasExecutor()) continue; Connection conn = neighbour_node.connection; neighbours.emplace_back(std::make_pair(resource, conn)); } @@ -57,13 +56,12 @@ get_neighbours_with_connetion(const ResourcePtr &self) { } void -Action::PushTaskToNeighbourRandomly(const TaskPtr &task, - const ResourcePtr &self) { +Action::PushTaskToNeighbourRandomly(const TaskPtr& task, const ResourcePtr& self) { auto neighbours = get_neighbours_with_connetion(self); if (not neighbours.empty()) { std::vector speeds; uint64_t total_speed = 0; - for (auto &neighbour : neighbours) { + for (auto& neighbour : neighbours) { uint64_t speed = neighbour.second.speed(); speeds.emplace_back(speed); total_speed += speed; @@ -83,33 +81,32 @@ Action::PushTaskToNeighbourRandomly(const TaskPtr &task, } } else { - //TODO: process + // TODO: process } } void -Action::PushTaskToAllNeighbour(const TaskPtr &task, const ResourcePtr &self) { +Action::PushTaskToAllNeighbour(const TaskPtr& task, const ResourcePtr& self) { auto neighbours = get_neighbours(self); - for (auto &neighbour : neighbours) { + for (auto& neighbour : neighbours) { neighbour->task_table().Put(task); } } void -Action::PushTaskToResource(const TaskPtr &task, const ResourcePtr &dest) { +Action::PushTaskToResource(const TaskPtr& task, const ResourcePtr& dest) { dest->task_table().Put(task); } void -Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, - ResourcePtr resource, +Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr resource, std::shared_ptr event) { if (not resource->HasExecutor() && event->task_table_item_->Move()) { auto task = event->task_table_item_->task; auto search_task = std::static_pointer_cast(task); bool moved = false; - //to support test task, REFACTOR + // to support test task, REFACTOR if (auto index_engine = search_task->index_engine_) { auto location = index_engine->GetLocation(); @@ -131,8 +128,7 @@ Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, } void -Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, - ResourcePtr resource, +Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr resource, std::shared_ptr event) { auto task = event->task_table_item_->task; if (resource->type() == ResourceType::DISK) { @@ -140,7 +136,7 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, auto compute_resources = res_mgr.lock()->GetComputeResources(); std::vector> paths; std::vector transport_costs; - for (auto &res : compute_resources) { + for (auto& res : compute_resources) { std::vector path; uint64_t transport_cost = ShortestPath(resource, res, res_mgr.lock(), path); transport_costs.push_back(transport_cost); @@ -155,8 +151,8 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, min_cost_idx = i; break; } - uint64_t cost = compute_resources[i]->TaskAvgCost() * compute_resources[i]->NumOfTaskToExec() - + transport_costs[i]; + uint64_t cost = + compute_resources[i]->TaskAvgCost() * compute_resources[i]->NumOfTaskToExec() + transport_costs[i]; if (min_cost > cost) { min_cost = cost; min_cost_idx = i; @@ -178,6 +174,6 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, } } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/Event.h b/cpp/src/scheduler/event/Event.h index 860c60c5b7..6d55820248 100644 --- a/cpp/src/scheduler/event/Event.h +++ b/cpp/src/scheduler/event/Event.h @@ -18,28 +18,20 @@ #pragma once #include -#include #include +#include namespace zilliz { namespace milvus { namespace scheduler { -enum class EventType { - START_UP, - LOAD_COMPLETED, - FINISH_TASK, - TASK_TABLE_UPDATED -}; +enum class EventType { START_UP, LOAD_COMPLETED, FINISH_TASK, TASK_TABLE_UPDATED }; class Resource; class Event { public: - explicit - Event(EventType type, std::weak_ptr resource) - : type_(type), - resource_(std::move(resource)) { + explicit Event(EventType type, std::weak_ptr resource) : type_(type), resource_(std::move(resource)) { } inline EventType @@ -50,7 +42,8 @@ class Event { virtual std::string Dump() const = 0; - friend std::ostream &operator<<(std::ostream &out, const Event &event); + friend std::ostream& + operator<<(std::ostream& out, const Event& event); public: EventType type_; @@ -59,6 +52,6 @@ class Event { using EventPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/EventDump.cpp b/cpp/src/scheduler/event/EventDump.cpp index a9ed751d88..6cc40ea28c 100644 --- a/cpp/src/scheduler/event/EventDump.cpp +++ b/cpp/src/scheduler/event/EventDump.cpp @@ -15,47 +15,46 @@ // specific language governing permissions and limitations // under the License. - #include "Event.h" -#include "StartUpEvent.h" -#include "LoadCompletedEvent.h" #include "FinishTaskEvent.h" +#include "LoadCompletedEvent.h" +#include "StartUpEvent.h" #include "TaskTableUpdatedEvent.h" namespace zilliz { namespace milvus { namespace scheduler { -std::ostream & -operator<<(std::ostream &out, const Event &event) { +std::ostream& +operator<<(std::ostream& out, const Event& event) { out << event.Dump(); return out; } -std::ostream & -operator<<(std::ostream &out, const StartUpEvent &event) { +std::ostream& +operator<<(std::ostream& out, const StartUpEvent& event) { out << event.Dump(); return out; } -std::ostream & -operator<<(std::ostream &out, const LoadCompletedEvent &event) { +std::ostream& +operator<<(std::ostream& out, const LoadCompletedEvent& event) { out << event.Dump(); return out; } -std::ostream & -operator<<(std::ostream &out, const FinishTaskEvent &event) { +std::ostream& +operator<<(std::ostream& out, const FinishTaskEvent& event) { out << event.Dump(); return out; } -std::ostream & -operator<<(std::ostream &out, const TaskTableUpdatedEvent &event) { +std::ostream& +operator<<(std::ostream& out, const TaskTableUpdatedEvent& event) { out << event.Dump(); return out; } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/FinishTaskEvent.h b/cpp/src/scheduler/event/FinishTaskEvent.h index f49acb16ad..b593abfe31 100644 --- a/cpp/src/scheduler/event/FinishTaskEvent.h +++ b/cpp/src/scheduler/event/FinishTaskEvent.h @@ -17,11 +17,12 @@ #pragma once +#include "scheduler/TaskTable.h" #include "scheduler/event/Event.h" #include -#include #include +#include namespace zilliz { namespace milvus { @@ -30,8 +31,7 @@ namespace scheduler { class FinishTaskEvent : public Event { public: FinishTaskEvent(std::weak_ptr resource, TaskTableItemPtr task_table_item) - : Event(EventType::FINISH_TASK, std::move(resource)), - task_table_item_(std::move(task_table_item)) { + : Event(EventType::FINISH_TASK, std::move(resource)), task_table_item_(std::move(task_table_item)) { } inline std::string @@ -39,12 +39,13 @@ class FinishTaskEvent : public Event { return ""; } - friend std::ostream &operator<<(std::ostream &out, const FinishTaskEvent &event); + friend std::ostream& + operator<<(std::ostream& out, const FinishTaskEvent& event); public: TaskTableItemPtr task_table_item_; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/LoadCompletedEvent.h b/cpp/src/scheduler/event/LoadCompletedEvent.h index 8d727f7435..fa1f34b8c1 100644 --- a/cpp/src/scheduler/event/LoadCompletedEvent.h +++ b/cpp/src/scheduler/event/LoadCompletedEvent.h @@ -17,12 +17,12 @@ #pragma once -#include "scheduler/event/Event.h" #include "scheduler/TaskTable.h" +#include "scheduler/event/Event.h" #include -#include #include +#include namespace zilliz { namespace milvus { @@ -31,8 +31,7 @@ namespace scheduler { class LoadCompletedEvent : public Event { public: LoadCompletedEvent(std::weak_ptr resource, TaskTableItemPtr task_table_item) - : Event(EventType::LOAD_COMPLETED, std::move(resource)), - task_table_item_(std::move(task_table_item)) { + : Event(EventType::LOAD_COMPLETED, std::move(resource)), task_table_item_(std::move(task_table_item)) { } inline std::string @@ -40,12 +39,13 @@ class LoadCompletedEvent : public Event { return ""; } - friend std::ostream &operator<<(std::ostream &out, const LoadCompletedEvent &event); + friend std::ostream& + operator<<(std::ostream& out, const LoadCompletedEvent& event); public: TaskTableItemPtr task_table_item_; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/StartUpEvent.h b/cpp/src/scheduler/event/StartUpEvent.h index 8e4ad120de..037bf0f4d0 100644 --- a/cpp/src/scheduler/event/StartUpEvent.h +++ b/cpp/src/scheduler/event/StartUpEvent.h @@ -20,8 +20,8 @@ #include "scheduler/event/Event.h" #include -#include #include +#include namespace zilliz { namespace milvus { @@ -29,8 +29,7 @@ namespace scheduler { class StartUpEvent : public Event { public: - explicit StartUpEvent(std::weak_ptr resource) - : Event(EventType::START_UP, std::move(resource)) { + explicit StartUpEvent(std::weak_ptr resource) : Event(EventType::START_UP, std::move(resource)) { } inline std::string @@ -38,9 +37,10 @@ class StartUpEvent : public Event { return ""; } - friend std::ostream &operator<<(std::ostream &out, const StartUpEvent &event); + friend std::ostream& + operator<<(std::ostream& out, const StartUpEvent& event); }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/TaskTableUpdatedEvent.h b/cpp/src/scheduler/event/TaskTableUpdatedEvent.h index ec579b31be..814b25f056 100644 --- a/cpp/src/scheduler/event/TaskTableUpdatedEvent.h +++ b/cpp/src/scheduler/event/TaskTableUpdatedEvent.h @@ -20,8 +20,8 @@ #include "Event.h" #include -#include #include +#include namespace zilliz { namespace milvus { @@ -38,9 +38,10 @@ class TaskTableUpdatedEvent : public Event { return ""; } - friend std::ostream &operator<<(std::ostream &out, const TaskTableUpdatedEvent &event); + friend std::ostream& + operator<<(std::ostream& out, const TaskTableUpdatedEvent& event); }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/DeleteJob.cpp b/cpp/src/scheduler/job/DeleteJob.cpp index 9d917751c6..5fcbb40513 100644 --- a/cpp/src/scheduler/job/DeleteJob.cpp +++ b/cpp/src/scheduler/job/DeleteJob.cpp @@ -23,10 +23,7 @@ namespace zilliz { namespace milvus { namespace scheduler { -DeleteJob::DeleteJob(JobId id, - std::string table_id, - engine::meta::MetaPtr meta_ptr, - uint64_t num_resource) +DeleteJob::DeleteJob(JobId id, std::string table_id, engine::meta::MetaPtr meta_ptr, uint64_t num_resource) : Job(id, JobType::DELETE), table_id_(std::move(table_id)), meta_ptr_(std::move(meta_ptr)), @@ -36,9 +33,7 @@ DeleteJob::DeleteJob(JobId id, void DeleteJob::WaitAndDelete() { std::unique_lock lock(mutex_); - cv_.wait(lock, [&] { - return done_resource == num_resource_; - }); + cv_.wait(lock, [&] { return done_resource == num_resource_; }); meta_ptr_->DeleteTableFiles(table_id_); } @@ -51,6 +46,6 @@ DeleteJob::ResourceDone() { cv_.notify_one(); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/DeleteJob.h b/cpp/src/scheduler/job/DeleteJob.h index 7d8b20e47c..6c2f1e40c7 100644 --- a/cpp/src/scheduler/job/DeleteJob.h +++ b/cpp/src/scheduler/job/DeleteJob.h @@ -16,16 +16,16 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include #include "Job.h" #include "db/meta/Meta.h" @@ -36,10 +36,7 @@ namespace scheduler { class DeleteJob : public Job { public: - DeleteJob(JobId id, - std::string table_id, - engine::meta::MetaPtr meta_ptr, - uint64_t num_resource); + DeleteJob(JobId id, std::string table_id, engine::meta::MetaPtr meta_ptr, uint64_t num_resource); public: void @@ -71,6 +68,6 @@ class DeleteJob : public Job { using DeleteJobPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/Job.h b/cpp/src/scheduler/job/Job.h index c646a4f034..ccfa3518f9 100644 --- a/cpp/src/scheduler/job/Job.h +++ b/cpp/src/scheduler/job/Job.h @@ -16,16 +16,16 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include namespace zilliz { namespace milvus { @@ -64,6 +64,6 @@ class Job { using JobPtr = std::shared_ptr; using JobWPtr = std::weak_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/SearchJob.cpp b/cpp/src/scheduler/job/SearchJob.cpp index dee7125fed..b4d3ef84c2 100644 --- a/cpp/src/scheduler/job/SearchJob.cpp +++ b/cpp/src/scheduler/job/SearchJob.cpp @@ -22,20 +22,13 @@ namespace zilliz { namespace milvus { namespace scheduler { -SearchJob::SearchJob(zilliz::milvus::scheduler::JobId id, - uint64_t topk, - uint64_t nq, - uint64_t nprobe, - const float *vectors) - : Job(id, JobType::SEARCH), - topk_(topk), - nq_(nq), - nprobe_(nprobe), - vectors_(vectors) { +SearchJob::SearchJob(zilliz::milvus::scheduler::JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, + const float* vectors) + : Job(id, JobType::SEARCH), topk_(topk), nq_(nq), nprobe_(nprobe), vectors_(vectors) { } bool -SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) { +SearchJob::AddIndexFile(const TableFileSchemaPtr& index_file) { std::unique_lock lock(mutex_); if (index_file == nullptr || index_files_.find(index_file->id_) != index_files_.end()) { return false; @@ -50,9 +43,7 @@ SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) { void SearchJob::WaitResult() { std::unique_lock lock(mutex_); - cv_.wait(lock, [this] { - return index_files_.empty(); - }); + cv_.wait(lock, [this] { return index_files_.empty(); }); SERVER_LOG_DEBUG << "SearchJob " << id() << " all done"; } @@ -64,16 +55,16 @@ SearchJob::SearchDone(size_t index_id) { SERVER_LOG_DEBUG << "SearchJob " << id() << " finish index file: " << index_id; } -ResultSet & +ResultSet& SearchJob::GetResult() { return result_; } -Status & +Status& SearchJob::GetStatus() { return status_; } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/SearchJob.h b/cpp/src/scheduler/job/SearchJob.h index 7bb7fbefbf..7ffb0cd201 100644 --- a/cpp/src/scheduler/job/SearchJob.h +++ b/cpp/src/scheduler/job/SearchJob.h @@ -16,17 +16,17 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include #include +#include #include "Job.h" #include "db/meta/MetaTypes.h" @@ -43,11 +43,11 @@ using ResultSet = std::vector; class SearchJob : public Job { public: - SearchJob(JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, const float *vectors); + SearchJob(JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, const float* vectors); public: bool - AddIndexFile(const TableFileSchemaPtr &index_file); + AddIndexFile(const TableFileSchemaPtr& index_file); void WaitResult(); @@ -55,10 +55,10 @@ class SearchJob : public Job { void SearchDone(size_t index_id); - ResultSet & + ResultSet& GetResult(); - Status & + Status& GetStatus(); public: @@ -77,12 +77,12 @@ class SearchJob : public Job { return nprobe_; } - const float * + const float* vectors() const { return vectors_; } - Id2IndexMap & + Id2IndexMap& index_files() { return index_files_; } @@ -92,7 +92,7 @@ class SearchJob : public Job { uint64_t nq_ = 0; uint64_t nprobe_ = 0; // TODO: smart pointer - const float *vectors_ = nullptr; + const float* vectors_ = nullptr; Id2IndexMap index_files_; // TODO: column-base better ? @@ -105,6 +105,6 @@ class SearchJob : public Job { using SearchJobPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Connection.h b/cpp/src/scheduler/resource/Connection.h index cf18b6c9a2..ac2ac81bf5 100644 --- a/cpp/src/scheduler/resource/Connection.h +++ b/cpp/src/scheduler/resource/Connection.h @@ -17,8 +17,8 @@ #pragma once -#include #include +#include #include namespace zilliz { @@ -28,11 +28,10 @@ namespace scheduler { class Connection { public: // TODO: update construct function, speed: double->uint64_t - Connection(std::string name, double speed) - : name_(std::move(name)), speed_(speed) { + Connection(std::string name, double speed) : name_(std::move(name)), speed_(speed) { } - const std::string & + const std::string& name() const { return name_; } @@ -60,6 +59,6 @@ class Connection { uint64_t speed_; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/CpuResource.cpp b/cpp/src/scheduler/resource/CpuResource.cpp index 5859dfd0cd..d1b0e2b8c3 100644 --- a/cpp/src/scheduler/resource/CpuResource.cpp +++ b/cpp/src/scheduler/resource/CpuResource.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/resource/CpuResource.h" #include @@ -24,8 +23,8 @@ namespace zilliz { namespace milvus { namespace scheduler { -std::ostream & -operator<<(std::ostream &out, const CpuResource &resource) { +std::ostream& +operator<<(std::ostream& out, const CpuResource& resource) { out << resource.Dump(); return out; } @@ -44,6 +43,6 @@ CpuResource::Process(TaskPtr task) { task->Execute(); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/CpuResource.h b/cpp/src/scheduler/resource/CpuResource.h index 2226523fdf..562a306373 100644 --- a/cpp/src/scheduler/resource/CpuResource.h +++ b/cpp/src/scheduler/resource/CpuResource.h @@ -27,15 +27,15 @@ namespace scheduler { class CpuResource : public Resource { public: - explicit - CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); + explicit CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); inline std::string Dump() const override { return ""; } - friend std::ostream &operator<<(std::ostream &out, const CpuResource &resource); + friend std::ostream& + operator<<(std::ostream& out, const CpuResource& resource); protected: void @@ -45,6 +45,6 @@ class CpuResource : public Resource { Process(TaskPtr task) override; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/DiskResource.cpp b/cpp/src/scheduler/resource/DiskResource.cpp index eee2424cc1..08c03d2ef2 100644 --- a/cpp/src/scheduler/resource/DiskResource.cpp +++ b/cpp/src/scheduler/resource/DiskResource.cpp @@ -24,8 +24,8 @@ namespace zilliz { namespace milvus { namespace scheduler { -std::ostream & -operator<<(std::ostream &out, const DiskResource &resource) { +std::ostream& +operator<<(std::ostream& out, const DiskResource& resource) { out << resource.Dump(); return out; } @@ -42,6 +42,6 @@ void DiskResource::Process(TaskPtr task) { } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/DiskResource.h b/cpp/src/scheduler/resource/DiskResource.h index a7caf5c662..d86de49934 100644 --- a/cpp/src/scheduler/resource/DiskResource.h +++ b/cpp/src/scheduler/resource/DiskResource.h @@ -27,15 +27,15 @@ namespace scheduler { class DiskResource : public Resource { public: - explicit - DiskResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); + explicit DiskResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); inline std::string Dump() const override { return ""; } - friend std::ostream &operator<<(std::ostream &out, const DiskResource &resource); + friend std::ostream& + operator<<(std::ostream& out, const DiskResource& resource); protected: void @@ -45,6 +45,6 @@ class DiskResource : public Resource { Process(TaskPtr task) override; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/GpuResource.cpp b/cpp/src/scheduler/resource/GpuResource.cpp index 3c7abc0b29..a28b00bbe5 100644 --- a/cpp/src/scheduler/resource/GpuResource.cpp +++ b/cpp/src/scheduler/resource/GpuResource.cpp @@ -15,15 +15,14 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/resource/GpuResource.h" namespace zilliz { namespace milvus { namespace scheduler { -std::ostream & -operator<<(std::ostream &out, const GpuResource &resource) { +std::ostream& +operator<<(std::ostream& out, const GpuResource& resource) { out << resource.Dump(); return out; } @@ -42,6 +41,6 @@ GpuResource::Process(TaskPtr task) { task->Execute(); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/GpuResource.h b/cpp/src/scheduler/resource/GpuResource.h index 9f19b07464..33a197aed0 100644 --- a/cpp/src/scheduler/resource/GpuResource.h +++ b/cpp/src/scheduler/resource/GpuResource.h @@ -28,15 +28,15 @@ namespace scheduler { class GpuResource : public Resource { public: - explicit - GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); + explicit GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); inline std::string Dump() const override { return ""; } - friend std::ostream &operator<<(std::ostream &out, const GpuResource &resource); + friend std::ostream& + operator<<(std::ostream& out, const GpuResource& resource); protected: void @@ -46,6 +46,6 @@ class GpuResource : public Resource { Process(TaskPtr task) override; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Node.cpp b/cpp/src/scheduler/resource/Node.cpp index cf652b8ba0..39333cc072 100644 --- a/cpp/src/scheduler/resource/Node.cpp +++ b/cpp/src/scheduler/resource/Node.cpp @@ -33,7 +33,7 @@ std::vector Node::GetNeighbours() { std::lock_guard lk(mutex_); std::vector ret; - for (auto &e : neighbours_) { + for (auto& e : neighbours_) { ret.push_back(e.second); } return ret; @@ -43,7 +43,7 @@ std::string Node::Dump() { std::stringstream ss; ss << "::neighbours:" << std::endl; - for (auto &neighbour : neighbours_) { + for (auto& neighbour : neighbours_) { ss << "\t" << std::endl; } @@ -51,7 +51,7 @@ Node::Dump() { } void -Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connection) { +Node::AddNeighbour(const NeighbourNodePtr& neighbour_node, Connection& connection) { std::lock_guard lk(mutex_); if (auto s = neighbour_node.lock()) { neighbours_.emplace(std::make_pair(s->id_, Neighbour(neighbour_node, connection))); @@ -59,6 +59,6 @@ Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connectio // else do nothing, consider it.. } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Node.h b/cpp/src/scheduler/resource/Node.h index 00337a7009..0713dafa2d 100644 --- a/cpp/src/scheduler/resource/Node.h +++ b/cpp/src/scheduler/resource/Node.h @@ -17,13 +17,13 @@ #pragma once -#include -#include #include +#include #include +#include -#include "scheduler/TaskTable.h" #include "Connection.h" +#include "scheduler/TaskTable.h" namespace zilliz { namespace milvus { @@ -34,8 +34,7 @@ class Node; using NeighbourNodePtr = std::weak_ptr; struct Neighbour { - Neighbour(NeighbourNodePtr nei, Connection conn) - : neighbour_node(nei), connection(conn) { + Neighbour(NeighbourNodePtr nei, Connection conn) : neighbour_node(nei), connection(conn) { } NeighbourNodePtr neighbour_node; @@ -48,7 +47,7 @@ class Node { Node(); void - AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connection); + AddNeighbour(const NeighbourNodePtr& neighbour_node, Connection& connection); std::vector GetNeighbours(); @@ -66,6 +65,6 @@ class Node { using NodePtr = std::shared_ptr; using NodeWPtr = std::weak_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Resource.cpp b/cpp/src/scheduler/resource/Resource.cpp index 4eb71b8ac5..8273d69d33 100644 --- a/cpp/src/scheduler/resource/Resource.cpp +++ b/cpp/src/scheduler/resource/Resource.cpp @@ -25,17 +25,13 @@ namespace zilliz { namespace milvus { namespace scheduler { -std::ostream & -operator<<(std::ostream &out, const Resource &resource) { +std::ostream& +operator<<(std::ostream& out, const Resource& resource) { out << resource.Dump(); return out; } -Resource::Resource(std::string name, - ResourceType type, - uint64_t device_id, - bool enable_loader, - bool enable_executor) +Resource::Resource(std::string name, ResourceType type, uint64_t device_id, bool enable_loader, bool enable_executor) : name_(std::move(name)), type_(type), device_id_(device_id), @@ -95,7 +91,7 @@ Resource::WakeupExecutor() { uint64_t Resource::NumOfTaskToExec() { uint64_t count = 0; - for (auto &task : task_table_) { + for (auto& task : task_table_) { if (task->state == TaskTableItemState::LOADED) ++count; } return count; @@ -106,8 +102,7 @@ Resource::pick_task_load() { auto indexes = task_table_.PickToLoad(10); for (auto index : indexes) { // try to set one task loading, then return - if (task_table_.Load(index)) - return task_table_.Get(index); + if (task_table_.Load(index)) return task_table_.Get(index); // else try next } return nullptr; @@ -118,8 +113,7 @@ Resource::pick_task_execute() { auto indexes = task_table_.PickToExecute(3); for (auto index : indexes) { // try to set one task executing, then return - if (task_table_.Execute(index)) - return task_table_.Get(index); + if (task_table_.Execute(index)) return task_table_.Get(index); // else try next } return nullptr; @@ -129,9 +123,7 @@ void Resource::loader_function() { while (running_) { std::unique_lock lock(load_mutex_); - load_cv_.wait(lock, [&] { - return load_flag_; - }); + load_cv_.wait(lock, [&] { return load_flag_; }); load_flag_ = false; lock.unlock(); while (true) { @@ -157,9 +149,7 @@ Resource::executor_function() { } while (running_) { std::unique_lock lock(exec_mutex_); - exec_cv_.wait(lock, [&] { - return exec_flag_; - }); + exec_cv_.wait(lock, [&] { return exec_flag_; }); exec_flag_ = false; lock.unlock(); while (true) { @@ -183,6 +173,6 @@ Resource::executor_function() { } } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Resource.h b/cpp/src/scheduler/resource/Resource.h index 1c18b1a2b2..f6ba88c2c3 100644 --- a/cpp/src/scheduler/resource/Resource.h +++ b/cpp/src/scheduler/resource/Resource.h @@ -17,20 +17,20 @@ #pragma once -#include -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include -#include "../event/Event.h" -#include "../event/StartUpEvent.h" -#include "../event/LoadCompletedEvent.h" -#include "../event/FinishTaskEvent.h" -#include "../event/TaskTableUpdatedEvent.h" #include "../TaskTable.h" +#include "../event/Event.h" +#include "../event/FinishTaskEvent.h" +#include "../event/LoadCompletedEvent.h" +#include "../event/StartUpEvent.h" +#include "../event/TaskTableUpdatedEvent.h" #include "../task/Task.h" #include "Connection.h" #include "Node.h" @@ -99,7 +99,7 @@ class Resource : public Node, public std::enable_shared_from_this { return device_id_; } - TaskTable & + TaskTable& task_table() { return task_table_; } @@ -130,14 +130,11 @@ class Resource : public Node, public std::enable_shared_from_this { return total_task_; } - friend std::ostream &operator<<(std::ostream &out, const Resource &resource); + friend std::ostream& + operator<<(std::ostream& out, const Resource& resource); protected: - Resource(std::string name, - ResourceType type, - uint64_t device_id, - bool enable_loader, - bool enable_executor); + Resource(std::string name, ResourceType type, uint64_t device_id, bool enable_loader, bool enable_executor); /* * Implementation by inherit class; @@ -212,6 +209,6 @@ class Resource : public Node, public std::enable_shared_from_this { using ResourcePtr = std::shared_ptr; using ResourceWPtr = std::weak_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/TestResource.cpp b/cpp/src/scheduler/resource/TestResource.cpp index 25560cf7ee..dd16117236 100644 --- a/cpp/src/scheduler/resource/TestResource.cpp +++ b/cpp/src/scheduler/resource/TestResource.cpp @@ -23,8 +23,8 @@ namespace zilliz { namespace milvus { namespace scheduler { -std::ostream & -operator<<(std::ostream &out, const TestResource &resource) { +std::ostream& +operator<<(std::ostream& out, const TestResource& resource) { out << resource.Dump(); return out; } @@ -43,6 +43,6 @@ TestResource::Process(TaskPtr task) { task->Execute(); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/TestResource.h b/cpp/src/scheduler/resource/TestResource.h index ac83a42c60..45dbe67500 100644 --- a/cpp/src/scheduler/resource/TestResource.h +++ b/cpp/src/scheduler/resource/TestResource.h @@ -19,8 +19,8 @@ #include "Resource.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -28,15 +28,15 @@ namespace scheduler { class TestResource : public Resource { public: - explicit - TestResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); + explicit TestResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); inline std::string Dump() const override { return ""; } - friend std::ostream &operator<<(std::ostream &out, const TestResource &resource); + friend std::ostream& + operator<<(std::ostream& out, const TestResource& resource); protected: void @@ -46,6 +46,6 @@ class TestResource : public Resource { Process(TaskPtr task) override; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/DeleteTask.cpp b/cpp/src/scheduler/task/DeleteTask.cpp index 52579d67c6..c6cc33ac88 100644 --- a/cpp/src/scheduler/task/DeleteTask.cpp +++ b/cpp/src/scheduler/task/DeleteTask.cpp @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/task/DeleteTask.h" namespace zilliz { namespace milvus { namespace scheduler { -XDeleteTask::XDeleteTask(const scheduler::DeleteJobPtr &delete_job) +XDeleteTask::XDeleteTask(const scheduler::DeleteJobPtr& delete_job) : Task(TaskType::DeleteTask), delete_job_(delete_job) { } @@ -35,6 +34,6 @@ XDeleteTask::Execute() { delete_job_->ResourceDone(); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/DeleteTask.h b/cpp/src/scheduler/task/DeleteTask.h index 608960e7c8..65e8724940 100644 --- a/cpp/src/scheduler/task/DeleteTask.h +++ b/cpp/src/scheduler/task/DeleteTask.h @@ -17,8 +17,8 @@ #pragma once -#include "scheduler/job/DeleteJob.h" #include "Task.h" +#include "scheduler/job/DeleteJob.h" namespace zilliz { namespace milvus { @@ -26,7 +26,7 @@ namespace scheduler { class XDeleteTask : public Task { public: - explicit XDeleteTask(const scheduler::DeleteJobPtr &job); + explicit XDeleteTask(const scheduler::DeleteJobPtr& job); void Load(LoadType type, uint8_t device_id) override; @@ -38,6 +38,6 @@ class XDeleteTask : public Task { scheduler::DeleteJobPtr delete_job_; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/Path.h b/cpp/src/scheduler/task/Path.h index 672dfff1b9..2be2a7fc0d 100644 --- a/cpp/src/scheduler/task/Path.h +++ b/cpp/src/scheduler/task/Path.h @@ -17,8 +17,8 @@ #pragma once -#include #include +#include namespace zilliz { namespace milvus { @@ -28,11 +28,11 @@ class Path { public: Path() = default; - Path(std::vector &path, uint64_t index) : path_(path), index_(index) { + Path(std::vector& path, uint64_t index) : path_(path), index_(index) { } void - push_back(const std::string &str) { + push_back(const std::string& str) { path_.push_back(str); } @@ -61,16 +61,17 @@ class Path { } public: - std::string & - operator[](uint64_t index) { + std::string& operator[](uint64_t index) { return path_[index]; } - std::vector::iterator begin() { + std::vector::iterator + begin() { return path_.begin(); } - std::vector::iterator end() { + std::vector::iterator + end() { return path_.end(); } @@ -79,6 +80,6 @@ class Path { uint64_t index_ = 0; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/SearchTask.cpp b/cpp/src/scheduler/task/SearchTask.cpp index 0c205fcafa..0255699915 100644 --- a/cpp/src/scheduler/task/SearchTask.cpp +++ b/cpp/src/scheduler/task/SearchTask.cpp @@ -16,15 +16,15 @@ // under the License. #include "scheduler/task/SearchTask.h" -#include "scheduler/job/SearchJob.h" -#include "metrics/Metrics.h" #include "db/engine/EngineFactory.h" -#include "utils/TimeRecorder.h" +#include "metrics/Metrics.h" +#include "scheduler/job/SearchJob.h" #include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include #include #include -#include namespace zilliz { namespace milvus { @@ -35,8 +35,8 @@ static constexpr size_t PARALLEL_REDUCE_BATCH = 1000; std::mutex XSearchTask::merge_mutex_; -//bool -//NeedParallelReduce(uint64_t nq, uint64_t topk) { +// bool +// NeedParallelReduce(uint64_t nq, uint64_t topk) { // server::ServerConfig &config = server::ServerConfig::GetInstance(); // server::ConfigNode &db_config = config.GetConfig(server::CONFIG_DB); // bool need_parallel = db_config.GetBoolValue(server::CONFIG_DB_PARALLEL_REDUCE, false); @@ -47,8 +47,8 @@ std::mutex XSearchTask::merge_mutex_; // return nq * topk >= PARALLEL_REDUCE_THRESHOLD; //} // -//void -//ParallelReduce(std::function &reduce_function, size_t max_index) { +// void +// ParallelReduce(std::function &reduce_function, size_t max_index) { // size_t reduce_batch = PARALLEL_REDUCE_BATCH; // // auto thread_count = std::thread::hardware_concurrency() - 1; //not all core do this work @@ -96,14 +96,10 @@ CollectFileMetrics(int file_type, size_t file_size) { } } -XSearchTask::XSearchTask(TableFileSchemaPtr file) - : Task(TaskType::SearchTask), file_(file) { +XSearchTask::XSearchTask(TableFileSchemaPtr file) : Task(TaskType::SearchTask), file_(file) { if (file_) { - index_engine_ = EngineFactory::Build(file_->dimension_, - file_->location_, - (EngineType) file_->engine_type_, - (MetricType) file_->metric_type_, - file_->nlist_); + index_engine_ = EngineFactory::Build(file_->dimension_, file_->location_, (EngineType)file_->engine_type_, + (MetricType)file_->metric_type_, file_->nlist_); } } @@ -128,8 +124,8 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { error_msg = "Wrong load type"; stat = Status(SERVER_UNEXPECTED_ERROR, error_msg); } - } catch (std::exception &ex) { - //typical error: out of disk space or permition denied + } catch (std::exception& ex) { + // typical error: out of disk space or permition denied error_msg = "Failed to load index file: " + std::string(ex.what()); stat = Status(SERVER_UNEXPECTED_ERROR, error_msg); } @@ -155,19 +151,20 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { size_t file_size = index_engine_->PhysicalSize(); - std::string info = "Load file id:" + std::to_string(file_->id_) + " file type:" + std::to_string(file_->file_type_) - + " size:" + std::to_string(file_size) + " bytes from location: " + file_->location_ + " totally cost"; + std::string info = "Load file id:" + std::to_string(file_->id_) + " file type:" + + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + + " bytes from location: " + file_->location_ + " totally cost"; double span = rc.ElapseFromBegin(info); -// for (auto &context : search_contexts_) { -// context->AccumLoadCost(span); -// } + // for (auto &context : search_contexts_) { + // context->AccumLoadCost(span); + // } CollectFileMetrics(file_->file_type_, file_size); - //step 2: return search task for later execution + // step 2: return search task for later execution index_id_ = file_->id_; index_type_ = file_->file_type_; -// search_contexts_.swap(search_contexts_); + // search_contexts_.swap(search_contexts_); } void @@ -176,8 +173,8 @@ XSearchTask::Execute() { return; } -// ENGINE_LOG_DEBUG << "Searching in file id:" << index_id_ << " with " -// << search_contexts_.size() << " tasks"; + // ENGINE_LOG_DEBUG << "Searching in file id:" << index_id_ << " with " + // << search_contexts_.size() << " tasks"; TimeRecorder rc("DoSearch file id:" + std::to_string(index_id_)); @@ -188,45 +185,43 @@ XSearchTask::Execute() { if (auto job = job_.lock()) { auto search_job = std::static_pointer_cast(job); - //step 1: allocate memory + // step 1: allocate memory uint64_t nq = search_job->nq(); uint64_t topk = search_job->topk(); uint64_t nprobe = search_job->nprobe(); - const float *vectors = search_job->vectors(); + const float* vectors = search_job->vectors(); output_ids.resize(topk * nq); output_distance.resize(topk * nq); - std::string hdr = "job " + std::to_string(search_job->id()) + - " nq " + std::to_string(nq) + - " topk " + std::to_string(topk); + std::string hdr = + "job " + std::to_string(search_job->id()) + " nq " + std::to_string(nq) + " topk " + std::to_string(topk); try { - //step 2: search + // step 2: search index_engine_->Search(nq, vectors, topk, nprobe, output_distance.data(), output_ids.data()); double span = rc.RecordSection(hdr + ", do search"); -// search_job->AccumSearchCost(span); + // search_job->AccumSearchCost(span); - - //step 3: cluster result + // step 3: cluster result scheduler::ResultSet result_set; auto spec_k = index_engine_->Count() < topk ? index_engine_->Count() : topk; XSearchTask::ClusterResult(output_ids, output_distance, nq, spec_k, result_set); span = rc.RecordSection(hdr + ", cluster result"); -// search_job->AccumReduceCost(span); + // search_job->AccumReduceCost(span); // step 4: pick up topk result XSearchTask::TopkResult(result_set, topk, metric_l2, search_job->GetResult()); span = rc.RecordSection(hdr + ", reduce topk"); -// search_job->AccumReduceCost(span); - } catch (std::exception &ex) { + // search_job->AccumReduceCost(span); + } catch (std::exception& ex) { ENGINE_LOG_ERROR << "SearchTask encounter exception: " << ex.what(); -// search_job->IndexSearchDone(index_id_);//mark as done avoid dead lock, even search failed + // search_job->IndexSearchDone(index_id_);//mark as done avoid dead lock, even search failed } - //step 5: notify to send result to client + // step 5: notify to send result to client search_job->SearchDone(index_id_); } @@ -237,14 +232,11 @@ XSearchTask::Execute() { } Status -XSearchTask::ClusterResult(const std::vector &output_ids, - const std::vector &output_distance, - uint64_t nq, - uint64_t topk, - scheduler::ResultSet &result_set) { +XSearchTask::ClusterResult(const std::vector& output_ids, const std::vector& output_distance, + uint64_t nq, uint64_t topk, scheduler::ResultSet& result_set) { if (output_ids.size() < nq * topk || output_distance.size() < nq * topk) { - std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + - " distance array size: " + std::to_string(output_distance.size()); + std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + " distance array size: " + + std::to_string(output_distance.size()); ENGINE_LOG_ERROR << msg; return Status(DB_ERROR, msg); } @@ -267,21 +259,19 @@ XSearchTask::ClusterResult(const std::vector &output_ids, } }; -// if (NeedParallelReduce(nq, topk)) { -// ParallelReduce(reduce_worker, nq); -// } else { + // if (NeedParallelReduce(nq, topk)) { + // ParallelReduce(reduce_worker, nq); + // } else { reduce_worker(0, nq); -// } + // } return Status::OK(); } Status -XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, - scheduler::Id2DistanceMap &distance_target, - uint64_t topk, - bool ascending) { - //Note: the score_src and score_target are already arranged by score in ascending order +XSearchTask::MergeResult(scheduler::Id2DistanceMap& distance_src, scheduler::Id2DistanceMap& distance_target, + uint64_t topk, bool ascending) { + // Note: the score_src and score_target are already arranged by score in ascending order if (distance_src.empty()) { ENGINE_LOG_WARNING << "Empty distance source array"; return Status::OK(); @@ -299,8 +289,8 @@ XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, distance_merged.reserve(topk); size_t src_index = 0, target_index = 0; while (true) { - //all score_src items are merged, if score_merged.size() still less than topk - //move items from score_target to score_merged until score_merged.size() equal topk + // all score_src items are merged, if score_merged.size() still less than topk + // move items from score_target to score_merged until score_merged.size() equal topk if (src_index >= src_count) { for (size_t i = target_index; i < target_count && distance_merged.size() < topk; ++i) { distance_merged.push_back(distance_target[i]); @@ -308,8 +298,8 @@ XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, break; } - //all score_target items are merged, if score_merged.size() still less than topk - //move items from score_src to score_merged until score_merged.size() equal topk + // all score_target items are merged, if score_merged.size() still less than topk + // move items from score_src to score_merged until score_merged.size() equal topk if (target_index >= target_count) { for (size_t i = src_index; i < src_count && distance_merged.size() < topk; ++i) { distance_merged.push_back(distance_src[i]); @@ -317,11 +307,11 @@ XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, break; } - //compare score, + // compare score, // if ascending = true, put smallest score to score_merged one by one // else, put largest score to score_merged one by one - auto &src_pair = distance_src[src_index]; - auto &target_pair = distance_target[target_index]; + auto& src_pair = distance_src[src_index]; + auto& target_pair = distance_target[target_index]; if (ascending) { if (src_pair.second > target_pair.second) { distance_merged.push_back(target_pair); @@ -340,7 +330,7 @@ XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, } } - //score_merged.size() already equal topk + // score_merged.size() already equal topk if (distance_merged.size() >= topk) { break; } @@ -352,10 +342,8 @@ XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, } Status -XSearchTask::TopkResult(scheduler::ResultSet &result_src, - uint64_t topk, - bool ascending, - scheduler::ResultSet &result_target) { +XSearchTask::TopkResult(scheduler::ResultSet& result_src, uint64_t topk, bool ascending, + scheduler::ResultSet& result_target) { if (result_target.empty()) { result_target.swap(result_src); return Status::OK(); @@ -369,21 +357,21 @@ XSearchTask::TopkResult(scheduler::ResultSet &result_src, std::function ReduceWorker = [&](size_t from_index, size_t to_index) { for (size_t i = from_index; i < to_index; i++) { - scheduler::Id2DistanceMap &score_src = result_src[i]; - scheduler::Id2DistanceMap &score_target = result_target[i]; + scheduler::Id2DistanceMap& score_src = result_src[i]; + scheduler::Id2DistanceMap& score_target = result_target[i]; XSearchTask::MergeResult(score_src, score_target, topk, ascending); } }; -// if (NeedParallelReduce(result_src.size(), topk)) { -// ParallelReduce(ReduceWorker, result_src.size()); -// } else { + // if (NeedParallelReduce(result_src.size(), topk)) { + // ParallelReduce(ReduceWorker, result_src.size()); + // } else { ReduceWorker(0, result_src.size()); -// } + // } return Status::OK(); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/SearchTask.h b/cpp/src/scheduler/task/SearchTask.h index 7c19ba20f9..872187a077 100644 --- a/cpp/src/scheduler/task/SearchTask.h +++ b/cpp/src/scheduler/task/SearchTask.h @@ -18,8 +18,8 @@ #pragma once #include "Task.h" -#include "scheduler/job/SearchJob.h" #include "scheduler/Definition.h" +#include "scheduler/job/SearchJob.h" #include @@ -39,21 +39,16 @@ class XSearchTask : public Task { Execute() override; public: - static Status ClusterResult(const std::vector &output_ids, - const std::vector &output_distence, - uint64_t nq, - uint64_t topk, - scheduler::ResultSet &result_set); + static Status + ClusterResult(const std::vector& output_ids, const std::vector& output_distence, uint64_t nq, + uint64_t topk, scheduler::ResultSet& result_set); - static Status MergeResult(scheduler::Id2DistanceMap &distance_src, - scheduler::Id2DistanceMap &distance_target, - uint64_t topk, - bool ascending); + static Status + MergeResult(scheduler::Id2DistanceMap& distance_src, scheduler::Id2DistanceMap& distance_target, uint64_t topk, + bool ascending); - static Status TopkResult(scheduler::ResultSet &result_src, - uint64_t topk, - bool ascending, - scheduler::ResultSet &result_target); + static Status + TopkResult(scheduler::ResultSet& result_src, uint64_t topk, bool ascending, scheduler::ResultSet& result_target); public: TableFileSchemaPtr file_; @@ -66,6 +61,6 @@ class XSearchTask : public Task { static std::mutex merge_mutex_; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/Task.h b/cpp/src/scheduler/task/Task.h index 3600c10f03..da5fb3b3af 100644 --- a/cpp/src/scheduler/task/Task.h +++ b/cpp/src/scheduler/task/Task.h @@ -17,13 +17,13 @@ #pragma once -#include "scheduler/tasklabel/TaskLabel.h" -#include "scheduler/job/Job.h" -#include "utils/Status.h" #include "Path.h" +#include "scheduler/job/Job.h" +#include "scheduler/tasklabel/TaskLabel.h" +#include "utils/Status.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -63,7 +63,7 @@ class Task { /* * Transport path; */ - inline Path & + inline Path& path() { return task_path_; } @@ -71,7 +71,7 @@ class Task { /* * Getter and Setter; */ - inline TaskLabelPtr & + inline TaskLabelPtr& label() { return label_; } @@ -85,12 +85,12 @@ class Task { public: Path task_path_; -// std::vector search_contexts_; + // std::vector search_contexts_; scheduler::JobWPtr job_; TaskType type_; TaskLabelPtr label_ = nullptr; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/TestTask.cpp b/cpp/src/scheduler/task/TestTask.cpp index fc66e56269..aba7343467 100644 --- a/cpp/src/scheduler/task/TestTask.cpp +++ b/cpp/src/scheduler/task/TestTask.cpp @@ -22,7 +22,7 @@ namespace zilliz { namespace milvus { namespace scheduler { -TestTask::TestTask(TableFileSchemaPtr &file) : XSearchTask(file) { +TestTask::TestTask(TableFileSchemaPtr& file) : XSearchTask(file) { } void @@ -43,11 +43,9 @@ TestTask::Execute() { void TestTask::Wait() { std::unique_lock lock(mutex_); - cv_.wait(lock, [&] { - return done_; - }); + cv_.wait(lock, [&] { return done_; }); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/TestTask.h b/cpp/src/scheduler/task/TestTask.h index 7051080861..5d96b888aa 100644 --- a/cpp/src/scheduler/task/TestTask.h +++ b/cpp/src/scheduler/task/TestTask.h @@ -25,7 +25,7 @@ namespace scheduler { class TestTask : public XSearchTask { public: - explicit TestTask(TableFileSchemaPtr &file); + explicit TestTask(TableFileSchemaPtr& file); public: void @@ -46,6 +46,6 @@ class TestTask : public XSearchTask { std::condition_variable cv_; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/tasklabel/BroadcastLabel.h b/cpp/src/scheduler/tasklabel/BroadcastLabel.h index 6fca107864..ed68d3ccc1 100644 --- a/cpp/src/scheduler/tasklabel/BroadcastLabel.h +++ b/cpp/src/scheduler/tasklabel/BroadcastLabel.h @@ -33,7 +33,6 @@ class BroadcastLabel : public TaskLabel { using BroadcastLabelPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/tasklabel/DefaultLabel.h b/cpp/src/scheduler/tasklabel/DefaultLabel.h index 7943c4f7c1..fe7019f655 100644 --- a/cpp/src/scheduler/tasklabel/DefaultLabel.h +++ b/cpp/src/scheduler/tasklabel/DefaultLabel.h @@ -33,6 +33,6 @@ class DefaultLabel : public TaskLabel { using DefaultLabelPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/tasklabel/SpecResLabel.h b/cpp/src/scheduler/tasklabel/SpecResLabel.h index cfc5aa9400..b5bea81b39 100644 --- a/cpp/src/scheduler/tasklabel/SpecResLabel.h +++ b/cpp/src/scheduler/tasklabel/SpecResLabel.h @@ -19,8 +19,8 @@ #include "TaskLabel.h" -#include #include +#include class Resource; @@ -32,16 +32,16 @@ namespace scheduler { class SpecResLabel : public TaskLabel { public: - explicit SpecResLabel(const ResourceWPtr &resource) + explicit SpecResLabel(const ResourceWPtr& resource) : TaskLabel(TaskLabelType::SPECIFIED_RESOURCE), resource_(resource) { } - inline ResourceWPtr & + inline ResourceWPtr& resource() { return resource_; } - inline std::string & + inline std::string& resource_name() { return resource_name_; } @@ -53,6 +53,6 @@ class SpecResLabel : public TaskLabel { using SpecResLabelPtr = std::shared_ptr(); -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/tasklabel/TaskLabel.h b/cpp/src/scheduler/tasklabel/TaskLabel.h index d8d404e094..57f486ea89 100644 --- a/cpp/src/scheduler/tasklabel/TaskLabel.h +++ b/cpp/src/scheduler/tasklabel/TaskLabel.h @@ -24,9 +24,9 @@ namespace milvus { namespace scheduler { enum class TaskLabelType { - DEFAULT, // means can be executed in any resource - SPECIFIED_RESOURCE, // means must executing in special resource - BROADCAST, // means all enable-executor resource must execute task + DEFAULT, // means can be executed in any resource + SPECIFIED_RESOURCE, // means must executing in special resource + BROADCAST, // means all enable-executor resource must execute task }; class TaskLabel { @@ -46,6 +46,6 @@ class TaskLabel { using TaskLabelPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/sdk/examples/grpcsimple/main.cpp b/cpp/src/sdk/examples/grpcsimple/main.cpp index 166707259a..76c60ac1f8 100644 --- a/cpp/src/sdk/examples/grpcsimple/main.cpp +++ b/cpp/src/sdk/examples/grpcsimple/main.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include #include #include @@ -24,10 +23,10 @@ #include "src/ClientTest.h" void -print_help(const std::string &app_name); +print_help(const std::string& app_name); int -main(int argc, char *argv[]) { +main(int argc, char* argv[]) { printf("Client start...\n"); std::string app_name = basename(argv[0]); @@ -44,19 +43,20 @@ main(int argc, char *argv[]) { while ((value = getopt_long(argc, argv, "s:p:h", long_options, &option_index)) != -1) { switch (value) { case 's': { - char *address_ptr = strdup(optarg); + char* address_ptr = strdup(optarg); address = address_ptr; free(address_ptr); break; } case 'p': { - char *port_ptr = strdup(optarg); + char* port_ptr = strdup(optarg); port = port_ptr; free(port_ptr); break; } case 'h': - default:print_help(app_name); + default: + print_help(app_name); return EXIT_SUCCESS; } } @@ -69,7 +69,7 @@ main(int argc, char *argv[]) { } void -print_help(const std::string &app_name) { +print_help(const std::string& app_name) { printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str()); printf(" Options:\n"); printf(" -s --server Server address, default 127.0.0.1\n"); diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp index e29bcc30f3..45ea707975 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp @@ -19,14 +19,14 @@ #include "MilvusApi.h" #include "cache/CpuCacheMgr.h" -#include #include -#include -#include #include +#include +#include #include -#include +#include #include +#include //#define SET_VECTOR_IDS; @@ -40,14 +40,14 @@ constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; constexpr int64_t BATCH_ROW_COUNT = 100000; constexpr int64_t NQ = 5; constexpr int64_t TOP_K = 10; -constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different +constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different constexpr int64_t ADD_VECTOR_LOOP = 1; constexpr int64_t SECONDS_EACH_HOUR = 3600; #define BLOCK_SPLITER std::cout << "===========================================" << std::endl; void -PrintTableSchema(const milvus::TableSchema &tb_schema) { +PrintTableSchema(const milvus::TableSchema& tb_schema) { BLOCK_SPLITER std::cout << "Table name: " << tb_schema.table_name << std::endl; std::cout << "Table dimension: " << tb_schema.dimension << std::endl; @@ -55,19 +55,18 @@ PrintTableSchema(const milvus::TableSchema &tb_schema) { } void -PrintSearchResult(const std::vector> &search_record_array, - const std::vector &topk_query_result_array) { +PrintSearchResult(const std::vector>& search_record_array, + const std::vector& topk_query_result_array) { BLOCK_SPLITER std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl; int32_t index = 0; - for (auto &result : topk_query_result_array) { + for (auto& result : topk_query_result_array) { auto search_id = search_record_array[index].first; index++; - std::cout << "No." << std::to_string(index) << " vector " << std::to_string(search_id) - << " top " << std::to_string(result.query_result_arrays.size()) - << " search result:" << std::endl; - for (auto &item : result.query_result_arrays) { + std::cout << "No." << std::to_string(index) << " vector " << std::to_string(search_id) << " top " + << std::to_string(result.query_result_arrays.size()) << " search result:" << std::endl; + for (auto& item : result.query_result_arrays) { std::cout << "\t" << std::to_string(item.id) << "\tdistance:" << std::to_string(item.distance); std::cout << std::endl; } @@ -84,9 +83,9 @@ CurrentTime() { tm t; gmtime_r(&tt, &t); - std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) - + "_" + std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) - + "_" + std::to_string(t.tm_min) + "_" + std::to_string(t.tm_sec); + std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) + "_" + + std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) + "_" + std::to_string(t.tm_min) + + "_" + std::to_string(t.tm_sec); return str; } @@ -100,8 +99,8 @@ CurrentTmDate(int64_t offset_day = 0) { tm t; gmtime_r(&tt, &t); - std::string str = std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1) - + "-" + std::to_string(t.tm_mday); + std::string str = + std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1) + "-" + std::to_string(t.tm_mday); return str; } @@ -124,8 +123,7 @@ BuildTableSchema() { } void -BuildVectors(int64_t from, int64_t to, - std::vector &vector_record_array) { +BuildVectors(int64_t from, int64_t to, std::vector& vector_record_array) { if (to <= from) { return; } @@ -135,7 +133,7 @@ BuildVectors(int64_t from, int64_t to, milvus::RowRecord record; record.data.resize(TABLE_DIMENSION); for (int64_t i = 0; i < TABLE_DIMENSION; i++) { - record.data[i] = (float) (k % (i + 1)); + record.data[i] = (float)(k % (i + 1)); } vector_record_array.emplace_back(record); @@ -150,8 +148,7 @@ Sleep(int seconds) { class TimeRecorder { public: - explicit TimeRecorder(const std::string &title) - : title_(title) { + explicit TimeRecorder(const std::string& title) : title_(title) { start_ = std::chrono::system_clock::now(); } @@ -167,16 +164,15 @@ class TimeRecorder { }; void -CheckResult(const std::vector> &search_record_array, - const std::vector &topk_query_result_array) { +CheckResult(const std::vector>& search_record_array, + const std::vector& topk_query_result_array) { BLOCK_SPLITER int64_t index = 0; - for (auto &result : topk_query_result_array) { + for (auto& result : topk_query_result_array) { auto result_id = result.query_result_arrays[0].id; auto search_id = search_record_array[index++].first; if (result_id != search_id) { - std::cout << "The top 1 result is wrong: " << result_id - << " vs. " << search_id << std::endl; + std::cout << "The top 1 result is wrong: " << result_id << " vs. " << search_id << std::endl; } else { std::cout << "Check result sucessfully" << std::endl; } @@ -186,8 +182,7 @@ CheckResult(const std::vector> &search_rec void DoSearch(std::shared_ptr conn, - const std::vector> &search_record_array, - const std::string &phase_name) { + const std::vector>& search_record_array, const std::string& phase_name) { std::vector query_range_array; milvus::Range rg; rg.start_value = CurrentTmDate(); @@ -195,7 +190,7 @@ DoSearch(std::shared_ptr conn, query_range_array.emplace_back(rg); std::vector record_array; - for (auto &pair : search_record_array) { + for (auto& pair : search_record_array) { record_array.push_back(pair.second); } @@ -214,24 +209,24 @@ DoSearch(std::shared_ptr conn, PrintSearchResult(search_record_array, topk_query_result_array); CheckResult(search_record_array, topk_query_result_array); } -} // namespace +} // namespace void -ClientTest::Test(const std::string &address, const std::string &port) { +ClientTest::Test(const std::string& address, const std::string& port) { std::shared_ptr conn = milvus::Connection::Create(); - {//connect server + { // connect server milvus::ConnectParam param = {address, port}; milvus::Status stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } - {//server version + { // server version std::string version = conn->ServerVersion(); std::cout << "Server version: " << version << std::endl; } - {//sdk version + { // sdk version std::string version = conn->ClientVersion(); std::cout << "SDK version: " << version << std::endl; } @@ -241,15 +236,15 @@ ClientTest::Test(const std::string &address, const std::string &port) { milvus::Status stat = conn->ShowTables(tables); std::cout << "ShowTables function call status: " << stat.message() << std::endl; std::cout << "All tables: " << std::endl; - for (auto &table : tables) { + for (auto& table : tables) { int64_t row_count = 0; -// conn->DropTable(table); + // conn->DropTable(table); stat = conn->CountTable(table, row_count); std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl; } } - {//create table + { // create table milvus::TableSchema tb_schema = BuildTableSchema(); milvus::Status stat = conn->CreateTable(tb_schema); std::cout << "CreateTable function call status: " << stat.message() << std::endl; @@ -261,7 +256,7 @@ ClientTest::Test(const std::string &address, const std::string &port) { } } - {//describe table + { // describe table milvus::TableSchema tb_schema; milvus::Status stat = conn->DescribeTable(TABLE_NAME, tb_schema); std::cout << "DescribeTable function call status: " << stat.message() << std::endl; @@ -269,21 +264,21 @@ ClientTest::Test(const std::string &address, const std::string &port) { } std::vector> search_record_array; - {//insert vectors - for (int i = 0; i < ADD_VECTOR_LOOP; i++) {//add vectors + { // insert vectors + for (int i = 0; i < ADD_VECTOR_LOOP; i++) { // add vectors std::vector record_array; int64_t begin_index = i * BATCH_ROW_COUNT; BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array); #ifdef SET_VECTOR_IDS record_ids.resize(ADD_VECTOR_LOOP * BATCH_ROW_COUNT); - for (auto j = begin_index; j record_ids; - //generate user defined ids + // generate user defined ids for (int k = 0; k < BATCH_ROW_COUNT; k++) { record_ids.push_back(i * BATCH_ROW_COUNT + k); } @@ -299,22 +294,21 @@ ClientTest::Test(const std::string &address, const std::string &port) { std::cout << "Returned id array count: " << record_ids.size() << std::endl; if (search_record_array.size() < NQ) { - search_record_array.push_back( - std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET])); + search_record_array.push_back(std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET])); } } } - {//search vectors without index + { // search vectors without index Sleep(2); int64_t row_count = 0; milvus::Status stat = conn->CountTable(TABLE_NAME, row_count); std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; -// DoSearch(conn, search_record_array, "Search without index"); + // DoSearch(conn, search_record_array, "Search without index"); } - {//wait unit build index finish + { // wait unit build index finish std::cout << "Wait until create all index done" << std::endl; milvus::IndexParam index; index.table_name = TABLE_NAME; @@ -328,19 +322,19 @@ ClientTest::Test(const std::string &address, const std::string &port) { std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; } - {//preload table + { // preload table milvus::Status stat = conn->PreloadTable(TABLE_NAME); std::cout << "PreloadTable function call status: " << stat.message() << std::endl; } - {//search vectors after build index finish + { // search vectors after build index finish for (uint64_t i = 0; i < 5; ++i) { DoSearch(conn, search_record_array, "Search after build index finish"); } -// std::cout << conn->DumpTaskTables() << std::endl; + // std::cout << conn->DumpTaskTables() << std::endl; } - {//delete index + { // delete index milvus::Status stat = conn->DropIndex(TABLE_NAME); std::cout << "DropIndex function call status: " << stat.message() << std::endl; @@ -349,7 +343,7 @@ ClientTest::Test(const std::string &address, const std::string &port) { std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; } - {//delete by range + { // delete by range milvus::Range rg; rg.start_value = CurrentTmDate(-2); rg.end_value = CurrentTmDate(-3); @@ -358,17 +352,18 @@ ClientTest::Test(const std::string &address, const std::string &port) { std::cout << "DeleteByRange function call status: " << stat.message() << std::endl; } - {//delete table -// Status stat = conn->DropTable(TABLE_NAME); -// std::cout << "DeleteTable function call status: " << stat.message() << std::endl; + { + // delete table + // Status stat = conn->DropTable(TABLE_NAME); + // std::cout << "DeleteTable function call status: " << stat.message() << std::endl; } - {//server status + { // server status std::string status = conn->ServerStatus(); std::cout << "Server status before disconnect: " << status << std::endl; } milvus::Connection::Destroy(conn); - {//server status + { // server status std::string status = conn->ServerStatus(); std::cout << "Server status after disconnect: " << status << std::endl; } diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h index 165c36a180..b028b63f44 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h @@ -21,5 +21,6 @@ class ClientTest { public: - void Test(const std::string &address, const std::string &port); + void + Test(const std::string& address, const std::string& port); }; diff --git a/cpp/src/sdk/grpc/ClientProxy.cpp b/cpp/src/sdk/grpc/ClientProxy.cpp index f4cd1e9a38..8ebd577e3a 100644 --- a/cpp/src/sdk/grpc/ClientProxy.cpp +++ b/cpp/src/sdk/grpc/ClientProxy.cpp @@ -20,14 +20,14 @@ #include "grpc/gen-milvus/milvus.grpc.pb.h" #include -#include #include +#include //#define GRPC_MULTIPLE_THREAD; namespace milvus { bool -UriCheck(const std::string &uri) { +UriCheck(const std::string& uri) { size_t index = uri.find_first_of(':', 0); if (index == std::string::npos) { return false; @@ -37,7 +37,7 @@ UriCheck(const std::string &uri) { } Status -ClientProxy::Connect(const ConnectParam ¶m) { +ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); @@ -53,7 +53,7 @@ ClientProxy::Connect(const ConnectParam ¶m) { } Status -ClientProxy::Connect(const std::string &uri) { +ClientProxy::Connect(const std::string& uri) { if (!UriCheck(uri)) { return Status(StatusCode::InvalidAgument, "Invalid uri"); } @@ -71,7 +71,7 @@ ClientProxy::Connected() const { try { std::string info; return client_ptr_->Cmd(info, ""); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::NotConnected, "connection lost: " + std::string(ex.what())); } } @@ -83,7 +83,7 @@ ClientProxy::Disconnect() { connected_ = false; channel_.reset(); return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "failed to disconnect: " + std::string(ex.what())); } } @@ -94,22 +94,22 @@ ClientProxy::ClientVersion() const { } Status -ClientProxy::CreateTable(const TableSchema ¶m) { +ClientProxy::CreateTable(const TableSchema& param) { try { ::milvus::grpc::TableSchema schema; schema.set_table_name(param.table_name); schema.set_dimension(param.dimension); schema.set_index_file_size(param.index_file_size); - schema.set_metric_type((int32_t) param.metric_type); + schema.set_metric_type((int32_t)param.metric_type); return client_ptr_->CreateTable(schema); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "failed to create table: " + std::string(ex.what())); } } bool -ClientProxy::HasTable(const std::string &table_name) { +ClientProxy::HasTable(const std::string& table_name) { Status status = Status::OK(); ::milvus::grpc::TableName grpc_table_name; grpc_table_name.set_table_name(table_name); @@ -118,34 +118,33 @@ ClientProxy::HasTable(const std::string &table_name) { } Status -ClientProxy::DropTable(const std::string &table_name) { +ClientProxy::DropTable(const std::string& table_name) { try { ::milvus::grpc::TableName grpc_table_name; grpc_table_name.set_table_name(table_name); return client_ptr_->DropTable(grpc_table_name); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "failed to drop table: " + std::string(ex.what())); } } Status -ClientProxy::CreateIndex(const IndexParam &index_param) { +ClientProxy::CreateIndex(const IndexParam& index_param) { try { - //TODO: add index params + // TODO: add index params ::milvus::grpc::IndexParam grpc_index_param; grpc_index_param.set_table_name(index_param.table_name); - grpc_index_param.mutable_index()->set_index_type((int32_t) index_param.index_type); + grpc_index_param.mutable_index()->set_index_type((int32_t)index_param.index_type); grpc_index_param.mutable_index()->set_nlist(index_param.nlist); return client_ptr_->CreateIndex(grpc_index_param); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "failed to build index: " + std::string(ex.what())); } } Status -ClientProxy::Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) { +ClientProxy::Insert(const std::string& table_name, const std::vector& record_array, + std::vector& id_array) { Status status = Status::OK(); try { //////////////////////////////////////////////////////////////////////////// @@ -154,20 +153,17 @@ ClientProxy::Insert(const std::string &table_name, int thread_count = 10; std::shared_ptr<::milvus::grpc::InsertInfos> insert_info_array( - new ::milvus::grpc::InsertInfos[thread_count], - std::default_delete<::milvus::grpc::InsertInfos[]>() ); + new ::milvus::grpc::InsertInfos[thread_count], std::default_delete<::milvus::grpc::InsertInfos[]>()); - std::shared_ptr<::milvus::grpc::VectorIds> vector_ids_array( - new ::milvus::grpc::VectorIds[thread_count], - std::default_delete<::milvus::grpc::VectorIds[]>() ); + std::shared_ptr<::milvus::grpc::VectorIds> vector_ids_array(new ::milvus::grpc::VectorIds[thread_count], + std::default_delete<::milvus::grpc::VectorIds[]>()); int64_t record_count = record_array.size() / thread_count; for (size_t i = 0; i < thread_count; i++) { insert_info_array.get()[i].set_table_name(table_name); for (size_t j = i * record_count; j < record_count * (i + 1); j++) { - ::milvus::grpc::RowRecord *grpc_record = - insert_info_array.get()[i].add_row_record_array(); + ::milvus::grpc::RowRecord* grpc_record = insert_info_array.get()[i].add_row_record_array(); for (size_t k = 0; k < record_array[j].data.size(); k++) { grpc_record->add_vector_data(record_array[j].data[k]); } @@ -177,16 +173,13 @@ ClientProxy::Insert(const std::string &table_name, std::cout << "*****************************************************\n"; auto start = std::chrono::high_resolution_clock::now(); for (size_t j = 0; j < thread_count; j++) { - threads.push_back( - std::thread(&GrpcClient::InsertVector, client_ptr_, - std::ref(vector_ids_array.get()[j]), std::ref(insert_info_array.get()[j]), - std::ref(status))); + threads.push_back(std::thread(&GrpcClient::InsertVector, client_ptr_, std::ref(vector_ids_array.get()[j]), + std::ref(insert_info_array.get()[j]), std::ref(status))); } std::for_each(threads.begin(), threads.end(), std::mem_fn(&std::thread::join)); auto finish = std::chrono::high_resolution_clock::now(); - std::cout << - "InsertVector cost: " << std::chrono::duration_cast>(finish - start).count() - << "s\n"; + std::cout << "InsertVector cost: " + << std::chrono::duration_cast>(finish - start).count() << "s\n"; std::cout << "*****************************************************\n"; for (size_t i = 0; i < thread_count; i++) { @@ -198,14 +191,14 @@ ClientProxy::Insert(const std::string &table_name, ::milvus::grpc::InsertParam insert_param; insert_param.set_table_name(table_name); - for (auto &record : record_array) { - ::milvus::grpc::RowRecord *grpc_record = insert_param.add_row_record_array(); + for (auto& record : record_array) { + ::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array(); for (size_t i = 0; i < record.data.size(); i++) { grpc_record->add_vector_data(record.data[i]); } } - //Single thread + // Single thread ::milvus::grpc::VectorIds vector_ids; if (!id_array.empty()) { for (auto i = 0; i < id_array.size(); i++) { @@ -219,7 +212,7 @@ ClientProxy::Insert(const std::string &table_name, } } #endif - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to add vector: " + std::string(ex.what())); } @@ -227,37 +220,34 @@ ClientProxy::Insert(const std::string &table_name, } Status -ClientProxy::Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) { +ClientProxy::Search(const std::string& table_name, const std::vector& query_record_array, + const std::vector& query_range_array, int64_t topk, int64_t nprobe, + std::vector& topk_query_result_array) { try { - //step 1: convert vectors data + // step 1: convert vectors data ::milvus::grpc::SearchParam search_param; search_param.set_table_name(table_name); search_param.set_topk(topk); search_param.set_nprobe(nprobe); - for (auto &record : query_record_array) { - ::milvus::grpc::RowRecord *row_record = search_param.add_query_record_array(); - for (auto &rec : record.data) { + for (auto& record : query_record_array) { + ::milvus::grpc::RowRecord* row_record = search_param.add_query_record_array(); + for (auto& rec : record.data) { row_record->add_vector_data(rec); } } - //step 2: convert range array - for (auto &range : query_range_array) { - ::milvus::grpc::Range *grpc_range = search_param.add_query_range_array(); + // step 2: convert range array + for (auto& range : query_range_array) { + ::milvus::grpc::Range* grpc_range = search_param.add_query_range_array(); grpc_range->set_start_value(range.start_value); grpc_range->set_end_value(range.end_value); } - //step 3: search vectors + // step 3: search vectors ::milvus::grpc::TopKQueryResultList topk_query_result_list; Status status = client_ptr_->Search(topk_query_result_list, search_param); - //step 4: convert result array + // step 4: convert result array for (uint64_t i = 0; i < topk_query_result_list.topk_query_result_size(); ++i) { TopKQueryResult result; for (uint64_t j = 0; j < topk_query_result_list.topk_query_result(i).query_result_arrays_size(); ++j) { @@ -270,13 +260,13 @@ ClientProxy::Search(const std::string &table_name, topk_query_result_array.emplace_back(result); } return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to search vectors: " + std::string(ex.what())); } } Status -ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_schema) { +ClientProxy::DescribeTable(const std::string& table_name, TableSchema& table_schema) { try { ::milvus::grpc::TableSchema grpc_schema; @@ -285,27 +275,27 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch table_schema.table_name = grpc_schema.table_name(); table_schema.dimension = grpc_schema.dimension(); table_schema.index_file_size = grpc_schema.index_file_size(); - table_schema.metric_type = (MetricType) grpc_schema.metric_type(); + table_schema.metric_type = (MetricType)grpc_schema.metric_type(); return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to describe table: " + std::string(ex.what())); } } Status -ClientProxy::CountTable(const std::string &table_name, int64_t &row_count) { +ClientProxy::CountTable(const std::string& table_name, int64_t& row_count) { try { Status status; row_count = client_ptr_->CountTable(table_name, status); return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to show tables: " + std::string(ex.what())); } } Status -ClientProxy::ShowTables(std::vector &table_array) { +ClientProxy::ShowTables(std::vector& table_array) { try { Status status; milvus::grpc::TableNameList table_name_list; @@ -316,7 +306,7 @@ ClientProxy::ShowTables(std::vector &table_array) { table_array[i] = table_name_list.table_names(i); } return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to show tables: " + std::string(ex.what())); } } @@ -328,7 +318,7 @@ ClientProxy::ServerVersion() const { std::string version; Status status = client_ptr_->Cmd(version, "version"); return version; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return ""; } } @@ -343,7 +333,7 @@ ClientProxy::ServerStatus() const { std::string dummy; Status status = client_ptr_->Cmd(dummy, ""); return "server alive"; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return "connection lost"; } } @@ -358,62 +348,62 @@ ClientProxy::DumpTaskTables() const { std::string dummy; Status status = client_ptr_->Cmd(dummy, "tasktable"); return dummy; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return "connection lost"; } } Status -ClientProxy::DeleteByRange(milvus::Range &range, const std::string &table_name) { +ClientProxy::DeleteByRange(milvus::Range& range, const std::string& table_name) { try { ::milvus::grpc::DeleteByRangeParam delete_by_range_param; delete_by_range_param.set_table_name(table_name); delete_by_range_param.mutable_range()->set_start_value(range.start_value); delete_by_range_param.mutable_range()->set_end_value(range.end_value); return client_ptr_->DeleteByRange(delete_by_range_param); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to delete by range: " + std::string(ex.what())); } } Status -ClientProxy::PreloadTable(const std::string &table_name) const { +ClientProxy::PreloadTable(const std::string& table_name) const { try { ::milvus::grpc::TableName grpc_table_name; grpc_table_name.set_table_name(table_name); Status status = client_ptr_->PreloadTable(grpc_table_name); return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to preload tables: " + std::string(ex.what())); } } Status -ClientProxy::DescribeIndex(const std::string &table_name, IndexParam &index_param) const { +ClientProxy::DescribeIndex(const std::string& table_name, IndexParam& index_param) const { try { ::milvus::grpc::TableName grpc_table_name; grpc_table_name.set_table_name(table_name); ::milvus::grpc::IndexParam grpc_index_param; Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param); - index_param.index_type = (IndexType) (grpc_index_param.mutable_index()->index_type()); + index_param.index_type = (IndexType)(grpc_index_param.mutable_index()->index_type()); index_param.nlist = grpc_index_param.mutable_index()->nlist(); return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to describe index: " + std::string(ex.what())); } } Status -ClientProxy::DropIndex(const std::string &table_name) const { +ClientProxy::DropIndex(const std::string& table_name) const { try { ::milvus::grpc::TableName grpc_table_name; grpc_table_name.set_table_name(table_name); Status status = client_ptr_->DropIndex(grpc_table_name); return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to drop index: " + std::string(ex.what())); } } -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/grpc/ClientProxy.h b/cpp/src/sdk/grpc/ClientProxy.h index 044ce44847..dbeacc1380 100644 --- a/cpp/src/sdk/grpc/ClientProxy.h +++ b/cpp/src/sdk/grpc/ClientProxy.h @@ -17,12 +17,12 @@ #pragma once -#include "MilvusApi.h" #include "GrpcClient.h" +#include "MilvusApi.h" -#include -#include #include +#include +#include namespace milvus { @@ -30,10 +30,10 @@ class ClientProxy : public Connection { public: // Implementations of the Connection interface Status - Connect(const ConnectParam ¶m) override; + Connect(const ConnectParam& param) override; Status - Connect(const std::string &uri) override; + Connect(const std::string& uri) override; Status Connected() const override; @@ -42,38 +42,34 @@ class ClientProxy : public Connection { Disconnect() override; Status - CreateTable(const TableSchema ¶m) override; + CreateTable(const TableSchema& param) override; bool - HasTable(const std::string &table_name) override; + HasTable(const std::string& table_name) override; Status - DropTable(const std::string &table_name) override; + DropTable(const std::string& table_name) override; Status - CreateIndex(const IndexParam &index_param) override; + CreateIndex(const IndexParam& index_param) override; Status - Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) override; + Insert(const std::string& table_name, const std::vector& record_array, + std::vector& id_array) override; Status - Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) override; + Search(const std::string& table_name, const std::vector& query_record_array, + const std::vector& query_range_array, int64_t topk, int64_t nprobe, + std::vector& topk_query_result_array) override; Status - DescribeTable(const std::string &table_name, TableSchema &table_schema) override; + DescribeTable(const std::string& table_name, TableSchema& table_schema) override; Status - CountTable(const std::string &table_name, int64_t &row_count) override; + CountTable(const std::string& table_name, int64_t& row_count) override; Status - ShowTables(std::vector &table_array) override; + ShowTables(std::vector& table_array) override; std::string ClientVersion() const override; @@ -88,17 +84,16 @@ class ClientProxy : public Connection { DumpTaskTables() const override; Status - DeleteByRange(Range &range, - const std::string &table_name) override; + DeleteByRange(Range& range, const std::string& table_name) override; Status - PreloadTable(const std::string &table_name) const override; + PreloadTable(const std::string& table_name) const override; Status - DescribeIndex(const std::string &table_name, IndexParam &index_param) const override; + DescribeIndex(const std::string& table_name, IndexParam& index_param) const override; Status - DropIndex(const std::string &table_name) const override; + DropIndex(const std::string& table_name) const override; private: std::shared_ptr<::grpc::Channel> channel_; @@ -108,4 +103,4 @@ class ClientProxy : public Connection { bool connected_ = false; }; -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/grpc/GrpcClient.cpp b/cpp/src/sdk/grpc/GrpcClient.cpp index df86f0a3e7..5c27c3b73f 100644 --- a/cpp/src/sdk/grpc/GrpcClient.cpp +++ b/cpp/src/sdk/grpc/GrpcClient.cpp @@ -23,9 +23,9 @@ #include #include -#include -#include #include +#include +#include using grpc::Channel; using grpc::ClientContext; @@ -35,14 +35,14 @@ using grpc::ClientWriter; using grpc::Status; namespace milvus { -GrpcClient::GrpcClient(std::shared_ptr<::grpc::Channel> &channel) +GrpcClient::GrpcClient(std::shared_ptr<::grpc::Channel>& channel) : stub_(::milvus::grpc::MilvusService::NewStub(channel)) { } GrpcClient::~GrpcClient() = default; Status -GrpcClient::CreateTable(const ::milvus::grpc::TableSchema &table_schema) { +GrpcClient::CreateTable(const ::milvus::grpc::TableSchema& table_schema) { ClientContext context; grpc::Status response; ::grpc::Status grpc_status = stub_->CreateTable(&context, table_schema, &response); @@ -60,8 +60,7 @@ GrpcClient::CreateTable(const ::milvus::grpc::TableSchema &table_schema) { } bool -GrpcClient::HasTable(const ::milvus::grpc::TableName &table_name, - Status &status) { +GrpcClient::HasTable(const ::milvus::grpc::TableName& table_name, Status& status) { ClientContext context; ::milvus::grpc::BoolReply response; ::grpc::Status grpc_status = stub_->HasTable(&context, table_name, &response); @@ -79,7 +78,7 @@ GrpcClient::HasTable(const ::milvus::grpc::TableName &table_name, } Status -GrpcClient::DropTable(const ::milvus::grpc::TableName &table_name) { +GrpcClient::DropTable(const ::milvus::grpc::TableName& table_name) { ClientContext context; grpc::Status response; ::grpc::Status grpc_status = stub_->DropTable(&context, table_name, &response); @@ -97,7 +96,7 @@ GrpcClient::DropTable(const ::milvus::grpc::TableName &table_name) { } Status -GrpcClient::CreateIndex(const ::milvus::grpc::IndexParam &index_param) { +GrpcClient::CreateIndex(const ::milvus::grpc::IndexParam& index_param) { ClientContext context; grpc::Status response; ::grpc::Status grpc_status = stub_->CreateIndex(&context, index_param, &response); @@ -115,9 +114,8 @@ GrpcClient::CreateIndex(const ::milvus::grpc::IndexParam &index_param) { } void -GrpcClient::Insert(::milvus::grpc::VectorIds &vector_ids, - const ::milvus::grpc::InsertParam &insert_param, - Status &status) { +GrpcClient::Insert(::milvus::grpc::VectorIds& vector_ids, const ::milvus::grpc::InsertParam& insert_param, + Status& status) { ClientContext context; ::grpc::Status grpc_status = stub_->Insert(&context, insert_param, &vector_ids); @@ -136,8 +134,8 @@ GrpcClient::Insert(::milvus::grpc::VectorIds &vector_ids, } Status -GrpcClient::Search(::milvus::grpc::TopKQueryResultList &topk_query_result_list, - const ::milvus::grpc::SearchParam &search_param) { +GrpcClient::Search(::milvus::grpc::TopKQueryResultList& topk_query_result_list, + const ::milvus::grpc::SearchParam& search_param) { ::milvus::grpc::TopKQueryResult query_result; ClientContext context; ::grpc::Status grpc_status = stub_->Search(&context, search_param, &topk_query_result_list); @@ -149,16 +147,14 @@ GrpcClient::Search(::milvus::grpc::TopKQueryResultList &topk_query_result_list, } if (topk_query_result_list.status().error_code() != grpc::SUCCESS) { std::cerr << topk_query_result_list.status().reason() << std::endl; - return Status(StatusCode::ServerFailed, - topk_query_result_list.status().reason()); + return Status(StatusCode::ServerFailed, topk_query_result_list.status().reason()); } return Status::OK(); } Status -GrpcClient::DescribeTable(::milvus::grpc::TableSchema &grpc_schema, - const std::string &table_name) { +GrpcClient::DescribeTable(::milvus::grpc::TableSchema& grpc_schema, const std::string& table_name) { ClientContext context; ::milvus::grpc::TableName grpc_tablename; grpc_tablename.set_table_name(table_name); @@ -172,15 +168,14 @@ GrpcClient::DescribeTable(::milvus::grpc::TableSchema &grpc_schema, if (grpc_schema.status().error_code() != grpc::SUCCESS) { std::cerr << grpc_schema.status().reason() << std::endl; - return Status(StatusCode::ServerFailed, - grpc_schema.status().reason()); + return Status(StatusCode::ServerFailed, grpc_schema.status().reason()); } return Status::OK(); } int64_t -GrpcClient::CountTable(const std::string &table_name, Status &status) { +GrpcClient::CountTable(const std::string& table_name, Status& status) { ClientContext context; ::milvus::grpc::TableRowCount response; ::milvus::grpc::TableName grpc_tablename; @@ -204,7 +199,7 @@ GrpcClient::CountTable(const std::string &table_name, Status &status) { } Status -GrpcClient::ShowTables(milvus::grpc::TableNameList &table_name_list) { +GrpcClient::ShowTables(milvus::grpc::TableNameList& table_name_list) { ClientContext context; ::milvus::grpc::Command command; ::grpc::Status grpc_status = stub_->ShowTables(&context, command, &table_name_list); @@ -217,16 +212,14 @@ GrpcClient::ShowTables(milvus::grpc::TableNameList &table_name_list) { if (table_name_list.status().error_code() != grpc::SUCCESS) { std::cerr << table_name_list.status().reason() << std::endl; - return Status(StatusCode::ServerFailed, - table_name_list.status().reason()); + return Status(StatusCode::ServerFailed, table_name_list.status().reason()); } return Status::OK(); } Status -GrpcClient::Cmd(std::string &result, - const std::string &cmd) { +GrpcClient::Cmd(std::string& result, const std::string& cmd) { ClientContext context; ::milvus::grpc::StringReply response; ::milvus::grpc::Command command; @@ -248,7 +241,7 @@ GrpcClient::Cmd(std::string &result, } Status -GrpcClient::PreloadTable(milvus::grpc::TableName &table_name) { +GrpcClient::PreloadTable(milvus::grpc::TableName& table_name) { ClientContext context; ::milvus::grpc::Status response; ::grpc::Status grpc_status = stub_->PreloadTable(&context, table_name, &response); @@ -266,7 +259,7 @@ GrpcClient::PreloadTable(milvus::grpc::TableName &table_name) { } Status -GrpcClient::DeleteByRange(grpc::DeleteByRangeParam &delete_by_range_param) { +GrpcClient::DeleteByRange(grpc::DeleteByRangeParam& delete_by_range_param) { ClientContext context; ::milvus::grpc::Status response; ::grpc::Status grpc_status = stub_->DeleteByRange(&context, delete_by_range_param, &response); @@ -290,7 +283,7 @@ GrpcClient::Disconnect() { } Status -GrpcClient::DescribeIndex(grpc::TableName &table_name, grpc::IndexParam &index_param) { +GrpcClient::DescribeIndex(grpc::TableName& table_name, grpc::IndexParam& index_param) { ClientContext context; ::grpc::Status grpc_status = stub_->DescribeIndex(&context, table_name, &index_param); @@ -307,7 +300,7 @@ GrpcClient::DescribeIndex(grpc::TableName &table_name, grpc::IndexParam &index_p } Status -GrpcClient::DropIndex(grpc::TableName &table_name) { +GrpcClient::DropIndex(grpc::TableName& table_name) { ClientContext context; ::milvus::grpc::Status response; ::grpc::Status grpc_status = stub_->DropIndex(&context, table_name, &response); @@ -324,4 +317,4 @@ GrpcClient::DropIndex(grpc::TableName &table_name) { return Status::OK(); } -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/grpc/GrpcClient.h b/cpp/src/sdk/grpc/GrpcClient.h index 8f81e83ae8..d2e6ae5095 100644 --- a/cpp/src/sdk/grpc/GrpcClient.h +++ b/cpp/src/sdk/grpc/GrpcClient.h @@ -37,56 +37,51 @@ namespace milvus { class GrpcClient { public: - explicit GrpcClient(std::shared_ptr<::grpc::Channel> &channel); + explicit GrpcClient(std::shared_ptr<::grpc::Channel>& channel); - virtual - ~GrpcClient(); + virtual ~GrpcClient(); Status - CreateTable(const grpc::TableSchema &table_schema); + CreateTable(const grpc::TableSchema& table_schema); bool - HasTable(const grpc::TableName &table_name, Status &status); + HasTable(const grpc::TableName& table_name, Status& status); Status - DropTable(const grpc::TableName &table_name); + DropTable(const grpc::TableName& table_name); Status - CreateIndex(const grpc::IndexParam &index_param); + CreateIndex(const grpc::IndexParam& index_param); void - Insert(grpc::VectorIds &vector_ids, - const grpc::InsertParam &insert_param, - Status &status); + Insert(grpc::VectorIds& vector_ids, const grpc::InsertParam& insert_param, Status& status); Status - Search(::milvus::grpc::TopKQueryResultList &topk_query_result_list, - const grpc::SearchParam &search_param); + Search(::milvus::grpc::TopKQueryResultList& topk_query_result_list, const grpc::SearchParam& search_param); Status - DescribeTable(grpc::TableSchema &grpc_schema, - const std::string &table_name); + DescribeTable(grpc::TableSchema& grpc_schema, const std::string& table_name); int64_t - CountTable(const std::string &table_name, Status &status); + CountTable(const std::string& table_name, Status& status); Status - ShowTables(milvus::grpc::TableNameList &table_name_list); + ShowTables(milvus::grpc::TableNameList& table_name_list); Status - Cmd(std::string &result, const std::string &cmd); + Cmd(std::string& result, const std::string& cmd); Status - DeleteByRange(grpc::DeleteByRangeParam &delete_by_range_param); + DeleteByRange(grpc::DeleteByRangeParam& delete_by_range_param); Status - PreloadTable(grpc::TableName &table_name); + PreloadTable(grpc::TableName& table_name); Status - DescribeIndex(grpc::TableName &table_name, grpc::IndexParam &index_param); + DescribeIndex(grpc::TableName& table_name, grpc::IndexParam& index_param); Status - DropIndex(grpc::TableName &table_name); + DropIndex(grpc::TableName& table_name); Status Disconnect(); @@ -95,4 +90,4 @@ class GrpcClient { std::unique_ptr stub_; }; -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index e6025fd52e..9425ef3be3 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "Status.h" +#include #include #include -#include /** \brief Milvus SDK namespace */ @@ -48,18 +47,18 @@ enum class MetricType { * @brief Connect API parameter */ struct ConnectParam { - std::string ip_address; ///< Server IP address - std::string port; ///< Server PORT + std::string ip_address; ///< Server IP address + std::string port; ///< Server PORT }; /** * @brief Table Schema */ struct TableSchema { - std::string table_name; ///< Table name - int64_t dimension = 0; ///< Vector dimension, must be a positive value - int64_t index_file_size = 0; ///< Index file size, must be a positive value - MetricType metric_type = MetricType::L2; ///< Index metric type + std::string table_name; ///< Table name + int64_t dimension = 0; ///< Vector dimension, must be a positive value + int64_t index_file_size = 0; ///< Index file size, must be a positive value + MetricType metric_type = MetricType::L2; ///< Index metric type }; /** @@ -67,30 +66,30 @@ struct TableSchema { * for DATE partition, the format is like: 'year-month-day' */ struct Range { - std::string start_value; ///< Range start - std::string end_value; ///< Range stop + std::string start_value; ///< Range start + std::string end_value; ///< Range stop }; /** * @brief Record inserted */ struct RowRecord { - std::vector data; ///< Vector raw data + std::vector data; ///< Vector raw data }; /** * @brief Query result */ struct QueryResult { - int64_t id; ///< Output result - double distance; ///< Vector similarity distance + int64_t id; ///< Output result + double distance; ///< Vector similarity distance }; /** * @brief TopK query result */ struct TopKQueryResult { - std::vector query_result_arrays; ///< TopK query result + std::vector query_result_arrays; ///< TopK query result }; /** @@ -129,7 +128,7 @@ class Connection { */ static Status - Destroy(std::shared_ptr &connection_ptr); + Destroy(std::shared_ptr& connection_ptr); /** * @brief Connect @@ -143,7 +142,7 @@ class Connection { */ virtual Status - Connect(const ConnectParam ¶m) = 0; + Connect(const ConnectParam& param) = 0; /** * @brief Connect @@ -156,7 +155,7 @@ class Connection { * @return Indicate if connect is successful */ virtual Status - Connect(const std::string &uri) = 0; + Connect(const std::string& uri) = 0; /** * @brief connected @@ -188,7 +187,7 @@ class Connection { * @return Indicate if table is created successfully */ virtual Status - CreateTable(const TableSchema ¶m) = 0; + CreateTable(const TableSchema& param) = 0; /** * @brief Test table existence method @@ -200,7 +199,7 @@ class Connection { * @return Indicate if table is cexist */ virtual bool - HasTable(const std::string &table_name) = 0; + HasTable(const std::string& table_name) = 0; /** * @brief Delete table method @@ -212,7 +211,7 @@ class Connection { * @return Indicate if table is delete successfully. */ virtual Status - DropTable(const std::string &table_name) = 0; + DropTable(const std::string& table_name) = 0; /** * @brief Create index method @@ -228,7 +227,7 @@ class Connection { * @return Indicate if build index successfully. */ virtual Status - CreateIndex(const IndexParam &index_param) = 0; + CreateIndex(const IndexParam& index_param) = 0; /** * @brief Add vector to table @@ -242,9 +241,8 @@ class Connection { * @return Indicate if vector array are inserted successfully */ virtual Status - Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) = 0; + Insert(const std::string& table_name, const std::vector& record_array, + std::vector& id_array) = 0; /** * @brief Search vector @@ -260,12 +258,9 @@ class Connection { * @return Indicate if query is successful. */ virtual Status - Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) = 0; + Search(const std::string& table_name, const std::vector& query_record_array, + const std::vector& query_range_array, int64_t topk, int64_t nprobe, + std::vector& topk_query_result_array) = 0; /** * @brief Show table description @@ -278,7 +273,7 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - DescribeTable(const std::string &table_name, TableSchema &table_schema) = 0; + DescribeTable(const std::string& table_name, TableSchema& table_schema) = 0; /** * @brief Get table row count @@ -291,8 +286,7 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - CountTable(const std::string &table_name, - int64_t &row_count) = 0; + CountTable(const std::string& table_name, int64_t& row_count) = 0; /** * @brief Show all tables in database @@ -304,7 +298,7 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - ShowTables(std::vector &table_array) = 0; + ShowTables(std::vector& table_array) = 0; /** * @brief Give the client version @@ -350,8 +344,7 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - DeleteByRange(Range &range, - const std::string &table_name) = 0; + DeleteByRange(Range& range, const std::string& table_name) = 0; /** * @brief preload table @@ -363,7 +356,7 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - PreloadTable(const std::string &table_name) const = 0; + PreloadTable(const std::string& table_name) const = 0; /** * @brief describe index @@ -375,7 +368,7 @@ class Connection { * @return index informations and indicate if this operation is successful. */ virtual Status - DescribeIndex(const std::string &table_name, IndexParam &index_param) const = 0; + DescribeIndex(const std::string& table_name, IndexParam& index_param) const = 0; /** * @brief drop index @@ -387,7 +380,7 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - DropIndex(const std::string &table_name) const = 0; + DropIndex(const std::string& table_name) const = 0; }; -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/include/Status.h b/cpp/src/sdk/include/Status.h index 670d9662b4..e9b267c5f7 100644 --- a/cpp/src/sdk/include/Status.h +++ b/cpp/src/sdk/include/Status.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include @@ -46,19 +45,19 @@ enum class StatusCode { */ class Status { public: - Status(StatusCode code, const std::string &msg); + Status(StatusCode code, const std::string& msg); Status(); ~Status(); - Status(const Status &s); + Status(const Status& s); - Status & - operator=(const Status &s); + Status& + operator=(const Status& s); - Status(Status &&s); + Status(Status&& s); - Status & - operator=(Status &&s); + Status& + operator=(Status&& s); static Status OK() { @@ -72,7 +71,7 @@ class Status { StatusCode code() const { - return (state_ == nullptr) ? StatusCode::OK : *(StatusCode *) (state_); + return (state_ == nullptr) ? StatusCode::OK : *(StatusCode*)(state_); } std::string @@ -80,13 +79,13 @@ class Status { private: inline void - CopyFrom(const Status &s); + CopyFrom(const Status& s); inline void - MoveFrom(Status &s); + MoveFrom(Status& s); private: - const char *state_ = nullptr; -}; // Status + const char* state_ = nullptr; +}; // Status -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/interface/ConnectionImpl.cpp b/cpp/src/sdk/interface/ConnectionImpl.cpp index 5a01cd1a79..7034ce4a4d 100644 --- a/cpp/src/sdk/interface/ConnectionImpl.cpp +++ b/cpp/src/sdk/interface/ConnectionImpl.cpp @@ -25,7 +25,7 @@ Connection::Create() { } Status -Connection::Destroy(std::shared_ptr &connection_ptr) { +Connection::Destroy(std::shared_ptr& connection_ptr) { if (connection_ptr != nullptr) { return connection_ptr->Disconnect(); } @@ -38,12 +38,12 @@ ConnectionImpl::ConnectionImpl() { } Status -ConnectionImpl::Connect(const ConnectParam ¶m) { +ConnectionImpl::Connect(const ConnectParam& param) { return client_proxy_->Connect(param); } Status -ConnectionImpl::Connect(const std::string &uri) { +ConnectionImpl::Connect(const std::string& uri) { return client_proxy_->Connect(uri); } @@ -63,55 +63,51 @@ ConnectionImpl::ClientVersion() const { } Status -ConnectionImpl::CreateTable(const TableSchema ¶m) { +ConnectionImpl::CreateTable(const TableSchema& param) { return client_proxy_->CreateTable(param); } bool -ConnectionImpl::HasTable(const std::string &table_name) { +ConnectionImpl::HasTable(const std::string& table_name) { return client_proxy_->HasTable(table_name); } Status -ConnectionImpl::DropTable(const std::string &table_name) { +ConnectionImpl::DropTable(const std::string& table_name) { return client_proxy_->DropTable(table_name); } Status -ConnectionImpl::CreateIndex(const IndexParam &index_param) { +ConnectionImpl::CreateIndex(const IndexParam& index_param) { return client_proxy_->CreateIndex(index_param); } Status -ConnectionImpl::Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) { +ConnectionImpl::Insert(const std::string& table_name, const std::vector& record_array, + std::vector& id_array) { return client_proxy_->Insert(table_name, record_array, id_array); } Status -ConnectionImpl::Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) { - return client_proxy_->Search(table_name, query_record_array, query_range_array, topk, - nprobe, topk_query_result_array); +ConnectionImpl::Search(const std::string& table_name, const std::vector& query_record_array, + const std::vector& query_range_array, int64_t topk, int64_t nprobe, + std::vector& topk_query_result_array) { + return client_proxy_->Search(table_name, query_record_array, query_range_array, topk, nprobe, + topk_query_result_array); } Status -ConnectionImpl::DescribeTable(const std::string &table_name, TableSchema &table_schema) { +ConnectionImpl::DescribeTable(const std::string& table_name, TableSchema& table_schema) { return client_proxy_->DescribeTable(table_name, table_schema); } Status -ConnectionImpl::CountTable(const std::string &table_name, int64_t &row_count) { +ConnectionImpl::CountTable(const std::string& table_name, int64_t& row_count) { return client_proxy_->CountTable(table_name, row_count); } Status -ConnectionImpl::ShowTables(std::vector &table_array) { +ConnectionImpl::ShowTables(std::vector& table_array) { return client_proxy_->ShowTables(table_array); } @@ -131,24 +127,23 @@ ConnectionImpl::DumpTaskTables() const { } Status -ConnectionImpl::DeleteByRange(Range &range, - const std::string &table_name) { +ConnectionImpl::DeleteByRange(Range& range, const std::string& table_name) { return client_proxy_->DeleteByRange(range, table_name); } Status -ConnectionImpl::PreloadTable(const std::string &table_name) const { +ConnectionImpl::PreloadTable(const std::string& table_name) const { return client_proxy_->PreloadTable(table_name); } Status -ConnectionImpl::DescribeIndex(const std::string &table_name, IndexParam &index_param) const { +ConnectionImpl::DescribeIndex(const std::string& table_name, IndexParam& index_param) const { return client_proxy_->DescribeIndex(table_name, index_param); } Status -ConnectionImpl::DropIndex(const std::string &table_name) const { +ConnectionImpl::DropIndex(const std::string& table_name) const { return client_proxy_->DropIndex(table_name); } -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/interface/ConnectionImpl.h b/cpp/src/sdk/interface/ConnectionImpl.h index 7d5e090882..6bc3432bc4 100644 --- a/cpp/src/sdk/interface/ConnectionImpl.h +++ b/cpp/src/sdk/interface/ConnectionImpl.h @@ -20,9 +20,9 @@ #include "MilvusApi.h" #include "sdk/grpc/ClientProxy.h" -#include #include #include +#include namespace milvus { @@ -32,10 +32,10 @@ class ConnectionImpl : public Connection { // Implementations of the Connection interface Status - Connect(const ConnectParam ¶m) override; + Connect(const ConnectParam& param) override; Status - Connect(const std::string &uri) override; + Connect(const std::string& uri) override; Status Connected() const override; @@ -44,38 +44,34 @@ class ConnectionImpl : public Connection { Disconnect() override; Status - CreateTable(const TableSchema ¶m) override; + CreateTable(const TableSchema& param) override; bool - HasTable(const std::string &table_name) override; + HasTable(const std::string& table_name) override; Status - DropTable(const std::string &table_name) override; + DropTable(const std::string& table_name) override; Status - CreateIndex(const IndexParam &index_param) override; + CreateIndex(const IndexParam& index_param) override; Status - Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) override; + Insert(const std::string& table_name, const std::vector& record_array, + std::vector& id_array) override; Status - Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) override; + Search(const std::string& table_name, const std::vector& query_record_array, + const std::vector& query_range_array, int64_t topk, int64_t nprobe, + std::vector& topk_query_result_array) override; Status - DescribeTable(const std::string &table_name, TableSchema &table_schema) override; + DescribeTable(const std::string& table_name, TableSchema& table_schema) override; Status - CountTable(const std::string &table_name, int64_t &row_count) override; + CountTable(const std::string& table_name, int64_t& row_count) override; Status - ShowTables(std::vector &table_array) override; + ShowTables(std::vector& table_array) override; std::string ClientVersion() const override; @@ -90,20 +86,19 @@ class ConnectionImpl : public Connection { DumpTaskTables() const override; Status - DeleteByRange(Range &range, - const std::string &table_name) override; + DeleteByRange(Range& range, const std::string& table_name) override; Status - PreloadTable(const std::string &table_name) const override; + PreloadTable(const std::string& table_name) const override; Status - DescribeIndex(const std::string &table_name, IndexParam &index_param) const override; + DescribeIndex(const std::string& table_name, IndexParam& index_param) const override; Status - DropIndex(const std::string &table_name) const override; + DropIndex(const std::string& table_name) const override; private: std::shared_ptr client_proxy_; }; -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/interface/Status.cpp b/cpp/src/sdk/interface/Status.cpp index a8780f2ddd..cd46c8c245 100644 --- a/cpp/src/sdk/interface/Status.cpp +++ b/cpp/src/sdk/interface/Status.cpp @@ -23,12 +23,12 @@ namespace milvus { constexpr int CODE_WIDTH = sizeof(StatusCode); -Status::Status(StatusCode code, const std::string &msg) { - //4 bytes store code - //4 bytes store message length - //the left bytes store message string - const uint32_t length = (uint32_t) msg.size(); - char *result = new char[length + sizeof(length) + CODE_WIDTH]; +Status::Status(StatusCode code, const std::string& msg) { + // 4 bytes store code + // 4 bytes store message length + // the left bytes store message string + const uint32_t length = (uint32_t)msg.size(); + char* result = new char[length + sizeof(length) + CODE_WIDTH]; memcpy(result, &code, CODE_WIDTH); memcpy(result + CODE_WIDTH, &length, sizeof(length)); memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length); @@ -36,38 +36,35 @@ Status::Status(StatusCode code, const std::string &msg) { state_ = result; } -Status::Status() - : state_(nullptr) { +Status::Status() : state_(nullptr) { } Status::~Status() { delete state_; } -Status::Status(const Status &s) - : state_(nullptr) { +Status::Status(const Status& s) : state_(nullptr) { CopyFrom(s); } -Status & -Status::operator=(const Status &s) { +Status& +Status::operator=(const Status& s) { CopyFrom(s); return *this; } -Status::Status(Status &&s) - : state_(nullptr) { +Status::Status(Status&& s) : state_(nullptr) { MoveFrom(s); } -Status & -Status::operator=(Status &&s) { +Status& +Status::operator=(Status&& s) { MoveFrom(s); return *this; } void -Status::CopyFrom(const Status &s) { +Status::CopyFrom(const Status& s) { delete state_; state_ = nullptr; if (s.state_ == nullptr) { @@ -78,11 +75,11 @@ Status::CopyFrom(const Status &s) { memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); int buff_len = length + sizeof(length) + CODE_WIDTH; state_ = new char[buff_len]; - memcpy((void *) state_, (void *) s.state_, buff_len); + memcpy((void*)state_, (void*)s.state_, buff_len); } void -Status::MoveFrom(Status &s) { +Status::MoveFrom(Status& s) { delete state_; state_ = s.state_; s.state_ = nullptr; @@ -104,6 +101,4 @@ Status::message() const { return msg; } -} // namespace milvus - - +} // namespace milvus diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 0ad122fcad..788f792feb 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -17,14 +17,14 @@ #include "server/Config.h" +#include #include #include #include -#include -#include #include -#include +#include #include +#include #include "config/ConfigMgr.h" #include "utils/CommonUtil.h" @@ -36,14 +36,14 @@ namespace server { constexpr uint64_t GB = 1UL << 30; -Config & +Config& Config::GetInstance() { static Config config_inst; return config_inst; } Status -Config::LoadConfigFile(const std::string &filename) { +Config::LoadConfigFile(const std::string& filename) { if (filename.empty()) { std::cerr << "ERROR: need specify config file" << std::endl; exit(1); @@ -56,14 +56,13 @@ Config::LoadConfigFile(const std::string &filename) { } try { - ConfigMgr *mgr = const_cast(ConfigMgr::GetInstance()); + ConfigMgr* mgr = const_cast(ConfigMgr::GetInstance()); ErrorCode err = mgr->LoadConfigFile(filename); if (err != 0) { std::cerr << "Server failed to load config file: " << filename << std::endl; exit(1); } - } - catch (YAML::Exception &e) { + } catch (YAML::Exception& e) { std::cerr << "Server failed to load config file: " << filename << std::endl; exit(1); } @@ -256,7 +255,7 @@ Config::ResetDefaultConfig() { } void -Config::PrintConfigSection(const std::string &config_node_name) { +Config::PrintConfigSection(const std::string& config_node_name) { std::cout << std::endl; std::cout << config_node_name << ":" << std::endl; if (config_map_.find(config_node_name) != config_map_.end()) { @@ -278,7 +277,7 @@ Config::PrintAll() { //////////////////////////////////////////////////////////////////////////////// Status -Config::CheckServerConfigAddress(const std::string &value) { +Config::CheckServerConfigAddress(const std::string& value) { if (!ValidationUtil::ValidateIpAddress(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config address: " + value); } @@ -286,7 +285,7 @@ Config::CheckServerConfigAddress(const std::string &value) { } Status -Config::CheckServerConfigPort(const std::string &value) { +Config::CheckServerConfigPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config port: " + value); } else { @@ -299,10 +298,8 @@ Config::CheckServerConfigPort(const std::string &value) { } Status -Config::CheckServerConfigDeployMode(const std::string &value) { - if (value != "single" && - value != "cluster_readonly" && - value != "cluster_writable") { +Config::CheckServerConfigDeployMode(const std::string& value) { + if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config mode [single, cluster_readonly, cluster_writable]: " + value); } @@ -310,7 +307,7 @@ Config::CheckServerConfigDeployMode(const std::string &value) { } Status -Config::CheckServerConfigTimeZone(const std::string &value) { +Config::CheckServerConfigTimeZone(const std::string& value) { if (value.length() <= 3) { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); } else { @@ -328,7 +325,7 @@ Config::CheckServerConfigTimeZone(const std::string &value) { } Status -Config::CheckDBConfigPrimaryPath(const std::string &value) { +Config::CheckDBConfigPrimaryPath(const std::string& value) { if (value.empty()) { return Status(SERVER_INVALID_ARGUMENT, "DB config primary_path empty"); } @@ -336,12 +333,12 @@ Config::CheckDBConfigPrimaryPath(const std::string &value) { } Status -Config::CheckDBConfigSecondaryPath(const std::string &value) { +Config::CheckDBConfigSecondaryPath(const std::string& value) { return Status::OK(); } Status -Config::CheckDBConfigBackendUrl(const std::string &value) { +Config::CheckDBConfigBackendUrl(const std::string& value) { if (!ValidationUtil::ValidateDbURI(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config backend_url: " + value); } @@ -349,7 +346,7 @@ Config::CheckDBConfigBackendUrl(const std::string &value) { } Status -Config::CheckDBConfigArchiveDiskThreshold(const std::string &value) { +Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config archive_disk_threshold: " + value); } @@ -357,7 +354,7 @@ Config::CheckDBConfigArchiveDiskThreshold(const std::string &value) { } Status -Config::CheckDBConfigArchiveDaysThreshold(const std::string &value) { +Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config archive_days_threshold: " + value); } @@ -365,7 +362,7 @@ Config::CheckDBConfigArchiveDaysThreshold(const std::string &value) { } Status -Config::CheckDBConfigInsertBufferSize(const std::string &value) { +Config::CheckDBConfigInsertBufferSize(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config insert_buffer_size: " + value); } else { @@ -380,7 +377,7 @@ Config::CheckDBConfigInsertBufferSize(const std::string &value) { } Status -Config::CheckDBConfigBuildIndexGPU(const std::string &value) { +Config::CheckDBConfigBuildIndexGPU(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value); } else { @@ -393,7 +390,7 @@ Config::CheckDBConfigBuildIndexGPU(const std::string &value) { } Status -Config::CheckMetricConfigEnableMonitor(const std::string &value) { +Config::CheckMetricConfigEnableMonitor(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config auto_bootup: " + value); } @@ -401,7 +398,7 @@ Config::CheckMetricConfigEnableMonitor(const std::string &value) { } Status -Config::CheckMetricConfigCollector(const std::string &value) { +Config::CheckMetricConfigCollector(const std::string& value) { if (value != "prometheus") { return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config collector: " + value); } @@ -409,7 +406,7 @@ Config::CheckMetricConfigCollector(const std::string &value) { } Status -Config::CheckMetricConfigPrometheusPort(const std::string &value) { +Config::CheckMetricConfigPrometheusPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config prometheus_port: " + value); } @@ -417,7 +414,7 @@ Config::CheckMetricConfigPrometheusPort(const std::string &value) { } Status -Config::CheckCacheConfigCpuCacheCapacity(const std::string &value) { +Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_cache_capacity: " + value); } else { @@ -426,7 +423,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string &value) { CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (cpu_cache_capacity >= total_mem) { return Status(SERVER_INVALID_ARGUMENT, "Cache config cpu_cache_capacity exceed system memory: " + value); - } else if (cpu_cache_capacity > (double) total_mem * 0.9) { + } else if (cpu_cache_capacity > (double)total_mem * 0.9) { std::cerr << "Warning: cpu_cache_capacity value is too big" << std::endl; } @@ -442,7 +439,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string &value) { } Status -Config::CheckCacheConfigCpuCacheThreshold(const std::string &value) { +Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_cache_threshold: " + value); } else { @@ -455,7 +452,7 @@ Config::CheckCacheConfigCpuCacheThreshold(const std::string &value) { } Status -Config::CheckCacheConfigGpuCacheCapacity(const std::string &value) { +Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_cache_capacity: " + value); } else { @@ -470,7 +467,7 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string &value) { } else if (gpu_cache_capacity >= gpu_memory) { return Status(SERVER_INVALID_ARGUMENT, "Cache config gpu_cache_capacity exceed GPU memory: " + std::to_string(gpu_memory)); - } else if (gpu_cache_capacity > (double) gpu_memory * 0.9) { + } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { std::cerr << "Warning: gpu_cache_capacity value is too big" << std::endl; } } @@ -478,7 +475,7 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string &value) { } Status -Config::CheckCacheConfigGpuCacheThreshold(const std::string &value) { +Config::CheckCacheConfigGpuCacheThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_cache_threshold: " + value); } else { @@ -491,7 +488,7 @@ Config::CheckCacheConfigGpuCacheThreshold(const std::string &value) { } Status -Config::CheckCacheConfigCacheInsertData(const std::string &value) { +Config::CheckCacheConfigCacheInsertData(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cache_insert_data: " + value); } @@ -499,7 +496,7 @@ Config::CheckCacheConfigCacheInsertData(const std::string &value) { } Status -Config::CheckEngineConfigUseBlasThreshold(const std::string &value) { +Config::CheckEngineConfigUseBlasThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config use_blas_threshold: " + value); } @@ -507,7 +504,7 @@ Config::CheckEngineConfigUseBlasThreshold(const std::string &value) { } Status -Config::CheckEngineConfigOmpThreadNum(const std::string &value) { +Config::CheckEngineConfigOmpThreadNum(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); } else { @@ -521,7 +518,7 @@ Config::CheckEngineConfigOmpThreadNum(const std::string &value) { } Status -Config::CheckResourceConfigMode(const std::string &value) { +Config::CheckResourceConfigMode(const std::string& value) { if (value != "simple") { return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config mode: " + value); } @@ -529,7 +526,7 @@ Config::CheckResourceConfigMode(const std::string &value) { } Status -Config::CheckResourceConfigPool(const std::vector &value) { +Config::CheckResourceConfigPool(const std::vector& value) { if (value.empty()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config pool"); } @@ -537,17 +534,15 @@ Config::CheckResourceConfigPool(const std::vector &value) { } //////////////////////////////////////////////////////////////////////////////// -ConfigNode & -Config::GetConfigNode(const std::string &name) { - ConfigMgr *mgr = ConfigMgr::GetInstance(); - ConfigNode &root_node = mgr->GetRootNode(); +ConfigNode& +Config::GetConfigNode(const std::string& name) { + ConfigMgr* mgr = ConfigMgr::GetInstance(); + ConfigNode& root_node = mgr->GetRootNode(); return root_node.GetChild(name); } Status -Config::GetConfigValueInMem(const std::string &parent_key, - const std::string &child_key, - std::string &value) { +Config::GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value) { std::lock_guard lock(mutex_); if (config_map_.find(parent_key) != config_map_.end() && config_map_[parent_key].find(child_key) != config_map_[parent_key].end()) { @@ -559,9 +554,7 @@ Config::GetConfigValueInMem(const std::string &parent_key, } void -Config::SetConfigValueInMem(const std::string &parent_key, - const std::string &child_key, - const std::string &value) { +Config::SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value) { std::lock_guard lock(mutex_); config_map_[parent_key][child_key] = value; } @@ -572,8 +565,7 @@ std::string Config::GetServerConfigStrAddress() { std::string value; if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS, - CONFIG_SERVER_ADDRESS_DEFAULT); + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); } return value; @@ -583,8 +575,7 @@ std::string Config::GetServerConfigStrPort() { std::string value; if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT, - CONFIG_SERVER_PORT_DEFAULT); + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); } return value; @@ -594,8 +585,7 @@ std::string Config::GetServerConfigStrDeployMode() { std::string value; if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_DEPLOY_MODE, - CONFIG_SERVER_DEPLOY_MODE_DEFAULT); + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_DEPLOY_MODE, CONFIG_SERVER_DEPLOY_MODE_DEFAULT); SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); } return value; @@ -605,8 +595,7 @@ std::string Config::GetServerConfigStrTimeZone() { std::string value; if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE, - CONFIG_SERVER_TIME_ZONE_DEFAULT); + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); } return value; @@ -618,8 +607,7 @@ std::string Config::GetDBConfigStrPrimaryPath() { std::string value; if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PRIMARY_PATH, - CONFIG_DB_PRIMARY_PATH_DEFAULT); + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PRIMARY_PATH, CONFIG_DB_PRIMARY_PATH_DEFAULT); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); } return value; @@ -629,8 +617,7 @@ std::string Config::GetDBConfigStrSecondaryPath() { std::string value; if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SECONDARY_PATH, - CONFIG_DB_SECONDARY_PATH_DEFAULT); + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SECONDARY_PATH, CONFIG_DB_SECONDARY_PATH_DEFAULT); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); } return value; @@ -640,8 +627,7 @@ std::string Config::GetDBConfigStrBackendUrl() { std::string value; if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL, - CONFIG_DB_BACKEND_URL_DEFAULT); + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL, CONFIG_DB_BACKEND_URL_DEFAULT); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); } return value; @@ -673,8 +659,7 @@ std::string Config::GetDBConfigStrInsertBufferSize() { std::string value; if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, - CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); } return value; @@ -684,8 +669,7 @@ std::string Config::GetDBConfigStrBuildIndexGPU() { std::string value; if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU, - CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); } return value; @@ -697,8 +681,8 @@ std::string Config::GetMetricConfigStrEnableMonitor() { std::string value; if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value).ok()) { - value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_ENABLE_MONITOR, - CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); + value = + GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_ENABLE_MONITOR, CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); } return value; @@ -708,8 +692,7 @@ std::string Config::GetMetricConfigStrCollector() { std::string value; if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { - value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR, - CONFIG_METRIC_COLLECTOR_DEFAULT); + value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); } return value; @@ -719,8 +702,8 @@ std::string Config::GetMetricConfigStrPrometheusPort() { std::string value; if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { - value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, - CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); + value = + GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); } return value; @@ -732,8 +715,8 @@ std::string Config::GetCacheConfigStrCpuCacheCapacity() { std::string value; if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value).ok()) { - value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_CACHE_CAPACITY, - CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); + value = GetConfigNode(CONFIG_CACHE) + .GetValue(CONFIG_CACHE_CPU_CACHE_CAPACITY, CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); } return value; @@ -743,8 +726,8 @@ std::string Config::GetCacheConfigStrCpuCacheThreshold() { std::string value; if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_CACHE_THRESHOLD, - CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT); + value = GetConfigNode(CONFIG_CACHE) + .GetValue(CONFIG_CACHE_CPU_CACHE_THRESHOLD, CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT); SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); } return value; @@ -754,8 +737,8 @@ std::string Config::GetCacheConfigStrGpuCacheCapacity() { std::string value; if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value).ok()) { - value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_CACHE_CAPACITY, - CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); + value = GetConfigNode(CONFIG_CACHE) + .GetValue(CONFIG_CACHE_GPU_CACHE_CAPACITY, CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); } return value; @@ -765,8 +748,8 @@ std::string Config::GetCacheConfigStrGpuCacheThreshold() { std::string value; if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_CACHE_THRESHOLD, - CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); + value = GetConfigNode(CONFIG_CACHE) + .GetValue(CONFIG_CACHE_GPU_CACHE_THRESHOLD, CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); } return value; @@ -776,8 +759,8 @@ std::string Config::GetCacheConfigStrCacheInsertData() { std::string value; if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { - value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, - CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); + value = GetConfigNode(CONFIG_CACHE) + .GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); } return value; @@ -789,8 +772,8 @@ std::string Config::GetEngineConfigStrUseBlasThreshold() { std::string value; if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_USE_BLAS_THRESHOLD, - CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); + value = GetConfigNode(CONFIG_ENGINE) + .GetValue(CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); } return value; @@ -800,8 +783,8 @@ std::string Config::GetEngineConfigStrOmpThreadNum() { std::string value; if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { - value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, - CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); + value = + GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); } return value; @@ -813,8 +796,7 @@ std::string Config::GetResourceConfigStrMode() { std::string value; if (!GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { - value = GetConfigNode(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE, - CONFIG_RESOURCE_MODE_DEFAULT); + value = GetConfigNode(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); } return value; @@ -822,49 +804,49 @@ Config::GetResourceConfigStrMode() { //////////////////////////////////////////////////////////////////////////////// Status -Config::GetServerConfigAddress(std::string &value) { +Config::GetServerConfigAddress(std::string& value) { value = GetServerConfigStrAddress(); return CheckServerConfigAddress(value); } Status -Config::GetServerConfigPort(std::string &value) { +Config::GetServerConfigPort(std::string& value) { value = GetServerConfigStrPort(); return CheckServerConfigPort(value); } Status -Config::GetServerConfigDeployMode(std::string &value) { +Config::GetServerConfigDeployMode(std::string& value) { value = GetServerConfigStrDeployMode(); return CheckServerConfigDeployMode(value); } Status -Config::GetServerConfigTimeZone(std::string &value) { +Config::GetServerConfigTimeZone(std::string& value) { value = GetServerConfigStrTimeZone(); return CheckServerConfigTimeZone(value); } Status -Config::GetDBConfigPrimaryPath(std::string &value) { +Config::GetDBConfigPrimaryPath(std::string& value) { value = GetDBConfigStrPrimaryPath(); return CheckDBConfigPrimaryPath(value); } Status -Config::GetDBConfigSecondaryPath(std::string &value) { +Config::GetDBConfigSecondaryPath(std::string& value) { value = GetDBConfigStrSecondaryPath(); return Status::OK(); } Status -Config::GetDBConfigBackendUrl(std::string &value) { +Config::GetDBConfigBackendUrl(std::string& value) { value = GetDBConfigStrBackendUrl(); return CheckDBConfigBackendUrl(value); } Status -Config::GetDBConfigArchiveDiskThreshold(int32_t &value) { +Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { std::string str = GetDBConfigStrArchiveDiskThreshold(); Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) return s; @@ -873,7 +855,7 @@ Config::GetDBConfigArchiveDiskThreshold(int32_t &value) { } Status -Config::GetDBConfigArchiveDaysThreshold(int32_t &value) { +Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { std::string str = GetDBConfigStrArchiveDaysThreshold(); Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) return s; @@ -882,7 +864,7 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t &value) { } Status -Config::GetDBConfigInsertBufferSize(int32_t &value) { +Config::GetDBConfigInsertBufferSize(int32_t& value) { std::string str = GetDBConfigStrInsertBufferSize(); Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) return s; @@ -891,7 +873,7 @@ Config::GetDBConfigInsertBufferSize(int32_t &value) { } Status -Config::GetDBConfigBuildIndexGPU(int32_t &value) { +Config::GetDBConfigBuildIndexGPU(int32_t& value) { std::string str = GetDBConfigStrBuildIndexGPU(); Status s = CheckDBConfigBuildIndexGPU(str); if (!s.ok()) return s; @@ -900,7 +882,7 @@ Config::GetDBConfigBuildIndexGPU(int32_t &value) { } Status -Config::GetMetricConfigEnableMonitor(bool &value) { +Config::GetMetricConfigEnableMonitor(bool& value) { std::string str = GetMetricConfigStrEnableMonitor(); Status s = CheckMetricConfigEnableMonitor(str); if (!s.ok()) return s; @@ -910,19 +892,19 @@ Config::GetMetricConfigEnableMonitor(bool &value) { } Status -Config::GetMetricConfigCollector(std::string &value) { +Config::GetMetricConfigCollector(std::string& value) { value = GetMetricConfigStrCollector(); return Status::OK(); } Status -Config::GetMetricConfigPrometheusPort(std::string &value) { +Config::GetMetricConfigPrometheusPort(std::string& value) { value = GetMetricConfigStrPrometheusPort(); return CheckMetricConfigPrometheusPort(value); } Status -Config::GetCacheConfigCpuCacheCapacity(int32_t &value) { +Config::GetCacheConfigCpuCacheCapacity(int32_t& value) { std::string str = GetCacheConfigStrCpuCacheCapacity(); Status s = CheckCacheConfigCpuCacheCapacity(str); if (!s.ok()) return s; @@ -931,7 +913,7 @@ Config::GetCacheConfigCpuCacheCapacity(int32_t &value) { } Status -Config::GetCacheConfigCpuCacheThreshold(float &value) { +Config::GetCacheConfigCpuCacheThreshold(float& value) { std::string str = GetCacheConfigStrCpuCacheThreshold(); Status s = CheckCacheConfigCpuCacheThreshold(str); if (!s.ok()) return s; @@ -940,7 +922,7 @@ Config::GetCacheConfigCpuCacheThreshold(float &value) { } Status -Config::GetCacheConfigGpuCacheCapacity(int32_t &value) { +Config::GetCacheConfigGpuCacheCapacity(int32_t& value) { std::string str = GetCacheConfigStrGpuCacheCapacity(); Status s = CheckCacheConfigGpuCacheCapacity(str); if (!s.ok()) return s; @@ -949,7 +931,7 @@ Config::GetCacheConfigGpuCacheCapacity(int32_t &value) { } Status -Config::GetCacheConfigGpuCacheThreshold(float &value) { +Config::GetCacheConfigGpuCacheThreshold(float& value) { std::string str = GetCacheConfigStrGpuCacheThreshold(); Status s = CheckCacheConfigGpuCacheThreshold(str); if (!s.ok()) return s; @@ -958,7 +940,7 @@ Config::GetCacheConfigGpuCacheThreshold(float &value) { } Status -Config::GetCacheConfigCacheInsertData(bool &value) { +Config::GetCacheConfigCacheInsertData(bool& value) { std::string str = GetCacheConfigStrCacheInsertData(); Status s = CheckCacheConfigCacheInsertData(str); if (!s.ok()) return s; @@ -968,7 +950,7 @@ Config::GetCacheConfigCacheInsertData(bool &value) { } Status -Config::GetEngineConfigUseBlasThreshold(int32_t &value) { +Config::GetEngineConfigUseBlasThreshold(int32_t& value) { std::string str = GetEngineConfigStrUseBlasThreshold(); Status s = CheckEngineConfigUseBlasThreshold(str); if (!s.ok()) return s; @@ -977,7 +959,7 @@ Config::GetEngineConfigUseBlasThreshold(int32_t &value) { } Status -Config::GetEngineConfigOmpThreadNum(int32_t &value) { +Config::GetEngineConfigOmpThreadNum(int32_t& value) { std::string str = GetEngineConfigStrOmpThreadNum(); Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) return s; @@ -986,13 +968,13 @@ Config::GetEngineConfigOmpThreadNum(int32_t &value) { } Status -Config::GetResourceConfigMode(std::string &value) { +Config::GetResourceConfigMode(std::string& value) { value = GetResourceConfigStrMode(); return CheckResourceConfigMode(value); } Status -Config::GetResourceConfigPool(std::vector &value) { +Config::GetResourceConfigPool(std::vector& value) { ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE); value = resource_config.GetSequence(CONFIG_RESOURCE_POOL); return CheckResourceConfigPool(value); @@ -1001,7 +983,7 @@ Config::GetResourceConfigPool(std::vector &value) { /////////////////////////////////////////////////////////////////////////////// /* server config */ Status -Config::SetServerConfigAddress(const std::string &value) { +Config::SetServerConfigAddress(const std::string& value) { Status s = CheckServerConfigAddress(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); @@ -1009,7 +991,7 @@ Config::SetServerConfigAddress(const std::string &value) { } Status -Config::SetServerConfigPort(const std::string &value) { +Config::SetServerConfigPort(const std::string& value) { Status s = CheckServerConfigPort(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); @@ -1017,7 +999,7 @@ Config::SetServerConfigPort(const std::string &value) { } Status -Config::SetServerConfigDeployMode(const std::string &value) { +Config::SetServerConfigDeployMode(const std::string& value) { Status s = CheckServerConfigDeployMode(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); @@ -1025,7 +1007,7 @@ Config::SetServerConfigDeployMode(const std::string &value) { } Status -Config::SetServerConfigTimeZone(const std::string &value) { +Config::SetServerConfigTimeZone(const std::string& value) { Status s = CheckServerConfigTimeZone(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); @@ -1034,7 +1016,7 @@ Config::SetServerConfigTimeZone(const std::string &value) { /* db config */ Status -Config::SetDBConfigPrimaryPath(const std::string &value) { +Config::SetDBConfigPrimaryPath(const std::string& value) { Status s = CheckDBConfigPrimaryPath(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); @@ -1042,7 +1024,7 @@ Config::SetDBConfigPrimaryPath(const std::string &value) { } Status -Config::SetDBConfigSecondaryPath(const std::string &value) { +Config::SetDBConfigSecondaryPath(const std::string& value) { Status s = CheckDBConfigSecondaryPath(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); @@ -1050,7 +1032,7 @@ Config::SetDBConfigSecondaryPath(const std::string &value) { } Status -Config::SetDBConfigBackendUrl(const std::string &value) { +Config::SetDBConfigBackendUrl(const std::string& value) { Status s = CheckDBConfigBackendUrl(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); @@ -1058,7 +1040,7 @@ Config::SetDBConfigBackendUrl(const std::string &value) { } Status -Config::SetDBConfigArchiveDiskThreshold(const std::string &value) { +Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { Status s = CheckDBConfigArchiveDiskThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); @@ -1066,7 +1048,7 @@ Config::SetDBConfigArchiveDiskThreshold(const std::string &value) { } Status -Config::SetDBConfigArchiveDaysThreshold(const std::string &value) { +Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { Status s = CheckDBConfigArchiveDaysThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); @@ -1074,7 +1056,7 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string &value) { } Status -Config::SetDBConfigInsertBufferSize(const std::string &value) { +Config::SetDBConfigInsertBufferSize(const std::string& value) { Status s = CheckDBConfigInsertBufferSize(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); @@ -1082,7 +1064,7 @@ Config::SetDBConfigInsertBufferSize(const std::string &value) { } Status -Config::SetDBConfigBuildIndexGPU(const std::string &value) { +Config::SetDBConfigBuildIndexGPU(const std::string& value) { Status s = CheckDBConfigBuildIndexGPU(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); @@ -1091,7 +1073,7 @@ Config::SetDBConfigBuildIndexGPU(const std::string &value) { /* metric config */ Status -Config::SetMetricConfigEnableMonitor(const std::string &value) { +Config::SetMetricConfigEnableMonitor(const std::string& value) { Status s = CheckMetricConfigEnableMonitor(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value); @@ -1099,7 +1081,7 @@ Config::SetMetricConfigEnableMonitor(const std::string &value) { } Status -Config::SetMetricConfigCollector(const std::string &value) { +Config::SetMetricConfigCollector(const std::string& value) { Status s = CheckMetricConfigCollector(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_COLLECTOR, value); @@ -1107,7 +1089,7 @@ Config::SetMetricConfigCollector(const std::string &value) { } Status -Config::SetMetricConfigPrometheusPort(const std::string &value) { +Config::SetMetricConfigPrometheusPort(const std::string& value) { Status s = CheckMetricConfigPrometheusPort(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_PROMETHEUS_PORT, value); @@ -1116,7 +1098,7 @@ Config::SetMetricConfigPrometheusPort(const std::string &value) { /* cache config */ Status -Config::SetCacheConfigCpuCacheCapacity(const std::string &value) { +Config::SetCacheConfigCpuCacheCapacity(const std::string& value) { Status s = CheckCacheConfigCpuCacheCapacity(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); @@ -1124,7 +1106,7 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string &value) { } Status -Config::SetCacheConfigCpuCacheThreshold(const std::string &value) { +Config::SetCacheConfigCpuCacheThreshold(const std::string& value) { Status s = CheckCacheConfigCpuCacheThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); @@ -1132,7 +1114,7 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string &value) { } Status -Config::SetCacheConfigGpuCacheCapacity(const std::string &value) { +Config::SetCacheConfigGpuCacheCapacity(const std::string& value) { Status s = CheckCacheConfigGpuCacheCapacity(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); @@ -1140,7 +1122,7 @@ Config::SetCacheConfigGpuCacheCapacity(const std::string &value) { } Status -Config::SetCacheConfigGpuCacheThreshold(const std::string &value) { +Config::SetCacheConfigGpuCacheThreshold(const std::string& value) { Status s = CheckCacheConfigGpuCacheThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); @@ -1148,7 +1130,7 @@ Config::SetCacheConfigGpuCacheThreshold(const std::string &value) { } Status -Config::SetCacheConfigCacheInsertData(const std::string &value) { +Config::SetCacheConfigCacheInsertData(const std::string& value) { Status s = CheckCacheConfigCacheInsertData(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CACHE_INSERT_DATA, value); @@ -1157,7 +1139,7 @@ Config::SetCacheConfigCacheInsertData(const std::string &value) { /* engine config */ Status -Config::SetEngineConfigUseBlasThreshold(const std::string &value) { +Config::SetEngineConfigUseBlasThreshold(const std::string& value) { Status s = CheckEngineConfigUseBlasThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); @@ -1165,7 +1147,7 @@ Config::SetEngineConfigUseBlasThreshold(const std::string &value) { } Status -Config::SetEngineConfigOmpThreadNum(const std::string &value) { +Config::SetEngineConfigOmpThreadNum(const std::string& value) { Status s = CheckEngineConfigOmpThreadNum(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_OMP_THREAD_NUM, value); @@ -1174,14 +1156,13 @@ Config::SetEngineConfigOmpThreadNum(const std::string &value) { /* resource config */ Status -Config::SetResourceConfigMode(const std::string &value) { +Config::SetResourceConfigMode(const std::string& value) { Status s = CheckResourceConfigMode(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_MODE, value); return Status::OK(); } -} // namespace server -} // namespace milvus -} // namespace zilliz - +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index c2484a850a..4e3d563901 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -17,252 +17,346 @@ #pragma once -#include -#include -#include -#include #include +#include +#include +#include +#include -#include "utils/Status.h" #include "config/ConfigNode.h" +#include "utils/Status.h" namespace zilliz { namespace milvus { namespace server { /* server config */ -static const char *CONFIG_SERVER = "server_config"; -static const char *CONFIG_SERVER_ADDRESS = "address"; -static const char *CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; -static const char *CONFIG_SERVER_PORT = "port"; -static const char *CONFIG_SERVER_PORT_DEFAULT = "19530"; -static const char *CONFIG_SERVER_DEPLOY_MODE = "deploy_mode"; -static const char *CONFIG_SERVER_DEPLOY_MODE_DEFAULT = "single"; -static const char *CONFIG_SERVER_TIME_ZONE = "time_zone"; -static const char *CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; +static const char* CONFIG_SERVER = "server_config"; +static const char* CONFIG_SERVER_ADDRESS = "address"; +static const char* CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; +static const char* CONFIG_SERVER_PORT = "port"; +static const char* CONFIG_SERVER_PORT_DEFAULT = "19530"; +static const char* CONFIG_SERVER_DEPLOY_MODE = "deploy_mode"; +static const char* CONFIG_SERVER_DEPLOY_MODE_DEFAULT = "single"; +static const char* CONFIG_SERVER_TIME_ZONE = "time_zone"; +static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; /* db config */ -static const char *CONFIG_DB = "db_config"; -static const char *CONFIG_DB_PRIMARY_PATH = "primary_path"; -static const char *CONFIG_DB_PRIMARY_PATH_DEFAULT = "/tmp/milvus"; -static const char *CONFIG_DB_SECONDARY_PATH = "secondary_path"; -static const char *CONFIG_DB_SECONDARY_PATH_DEFAULT = ""; -static const char *CONFIG_DB_BACKEND_URL = "backend_url"; -static const char *CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; -static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; -static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; -static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; -static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; -static const char *CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; -static const char *CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4"; -static const char *CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; -static const char *CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; +static const char* CONFIG_DB = "db_config"; +static const char* CONFIG_DB_PRIMARY_PATH = "primary_path"; +static const char* CONFIG_DB_PRIMARY_PATH_DEFAULT = "/tmp/milvus"; +static const char* CONFIG_DB_SECONDARY_PATH = "secondary_path"; +static const char* CONFIG_DB_SECONDARY_PATH_DEFAULT = ""; +static const char* CONFIG_DB_BACKEND_URL = "backend_url"; +static const char* CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; +static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; +static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; +static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; +static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; +static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; +static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4"; +static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; +static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; /* cache config */ -static const char *CONFIG_CACHE = "cache_config"; -static const char *CONFIG_CACHE_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; -static const char *CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "16"; -static const char *CONFIG_CACHE_GPU_CACHE_CAPACITY = "gpu_cache_capacity"; -static const char *CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT = "0"; -static const char *CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_mem_threshold"; -static const char *CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT = "0.85"; -static const char *CONFIG_CACHE_GPU_CACHE_THRESHOLD = "gpu_mem_threshold"; -static const char *CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT = "0.85"; -static const char *CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; -static const char *CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; +static const char* CONFIG_CACHE = "cache_config"; +static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; +static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "16"; +static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY = "gpu_cache_capacity"; +static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT = "0"; +static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_mem_threshold"; +static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT = "0.85"; +static const char* CONFIG_CACHE_GPU_CACHE_THRESHOLD = "gpu_mem_threshold"; +static const char* CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT = "0.85"; +static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; +static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; /* metric config */ -static const char *CONFIG_METRIC = "metric_config"; -static const char *CONFIG_METRIC_ENABLE_MONITOR = "enable_monitor"; -static const char *CONFIG_METRIC_ENABLE_MONITOR_DEFAULT = "false"; -static const char *CONFIG_METRIC_COLLECTOR = "collector"; -static const char *CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; -static const char *CONFIG_METRIC_PROMETHEUS = "prometheus_config"; -static const char *CONFIG_METRIC_PROMETHEUS_PORT = "port"; -static const char *CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "8080"; +static const char* CONFIG_METRIC = "metric_config"; +static const char* CONFIG_METRIC_ENABLE_MONITOR = "enable_monitor"; +static const char* CONFIG_METRIC_ENABLE_MONITOR_DEFAULT = "false"; +static const char* CONFIG_METRIC_COLLECTOR = "collector"; +static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; +static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config"; +static const char* CONFIG_METRIC_PROMETHEUS_PORT = "port"; +static const char* CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "8080"; /* engine config */ -static const char *CONFIG_ENGINE = "engine_config"; -static const char *CONFIG_ENGINE_USE_BLAS_THRESHOLD = "use_blas_threshold"; -static const char *CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT = "20"; -static const char *CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num"; -static const char *CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; +static const char* CONFIG_ENGINE = "engine_config"; +static const char* CONFIG_ENGINE_USE_BLAS_THRESHOLD = "use_blas_threshold"; +static const char* CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT = "20"; +static const char* CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num"; +static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; /* resource config */ -static const char *CONFIG_RESOURCE = "resource_config"; -static const char *CONFIG_RESOURCE_MODE = "mode"; -static const char *CONFIG_RESOURCE_MODE_DEFAULT = "simple"; -static const char *CONFIG_RESOURCE_POOL = "resource_pool"; +static const char* CONFIG_RESOURCE = "resource_config"; +static const char* CONFIG_RESOURCE_MODE = "mode"; +static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; +static const char* CONFIG_RESOURCE_POOL = "resource_pool"; class Config { public: - static Config &GetInstance(); - Status LoadConfigFile(const std::string &filename); - Status ValidateConfig(); - Status ResetDefaultConfig(); - void PrintAll(); + static Config& + GetInstance(); + Status + LoadConfigFile(const std::string& filename); + Status + ValidateConfig(); + Status + ResetDefaultConfig(); + void + PrintAll(); private: - ConfigNode &GetConfigNode(const std::string &name); + ConfigNode& + GetConfigNode(const std::string& name); - Status GetConfigValueInMem(const std::string &parent_key, - const std::string &child_key, - std::string &value); + Status + GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value); - void SetConfigValueInMem(const std::string &parent_key, - const std::string &child_key, - const std::string &value); + void + SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value); - void PrintConfigSection(const std::string &config_node_name); + void + PrintConfigSection(const std::string& config_node_name); /////////////////////////////////////////////////////////////////////////// /* server config */ - Status CheckServerConfigAddress(const std::string &value); - Status CheckServerConfigPort(const std::string &value); - Status CheckServerConfigDeployMode(const std::string &value); - Status CheckServerConfigTimeZone(const std::string &value); + Status + CheckServerConfigAddress(const std::string& value); + Status + CheckServerConfigPort(const std::string& value); + Status + CheckServerConfigDeployMode(const std::string& value); + Status + CheckServerConfigTimeZone(const std::string& value); /* db config */ - Status CheckDBConfigPrimaryPath(const std::string &value); - Status CheckDBConfigSecondaryPath(const std::string &value); - Status CheckDBConfigBackendUrl(const std::string &value); - Status CheckDBConfigArchiveDiskThreshold(const std::string &value); - Status CheckDBConfigArchiveDaysThreshold(const std::string &value); - Status CheckDBConfigInsertBufferSize(const std::string &value); - Status CheckDBConfigBuildIndexGPU(const std::string &value); + Status + CheckDBConfigPrimaryPath(const std::string& value); + Status + CheckDBConfigSecondaryPath(const std::string& value); + Status + CheckDBConfigBackendUrl(const std::string& value); + Status + CheckDBConfigArchiveDiskThreshold(const std::string& value); + Status + CheckDBConfigArchiveDaysThreshold(const std::string& value); + Status + CheckDBConfigInsertBufferSize(const std::string& value); + Status + CheckDBConfigBuildIndexGPU(const std::string& value); /* metric config */ - Status CheckMetricConfigEnableMonitor(const std::string &value); - Status CheckMetricConfigCollector(const std::string &value); - Status CheckMetricConfigPrometheusPort(const std::string &value); + Status + CheckMetricConfigEnableMonitor(const std::string& value); + Status + CheckMetricConfigCollector(const std::string& value); + Status + CheckMetricConfigPrometheusPort(const std::string& value); /* cache config */ - Status CheckCacheConfigCpuCacheCapacity(const std::string &value); - Status CheckCacheConfigCpuCacheThreshold(const std::string &value); - Status CheckCacheConfigGpuCacheCapacity(const std::string &value); - Status CheckCacheConfigGpuCacheThreshold(const std::string &value); - Status CheckCacheConfigCacheInsertData(const std::string &value); + Status + CheckCacheConfigCpuCacheCapacity(const std::string& value); + Status + CheckCacheConfigCpuCacheThreshold(const std::string& value); + Status + CheckCacheConfigGpuCacheCapacity(const std::string& value); + Status + CheckCacheConfigGpuCacheThreshold(const std::string& value); + Status + CheckCacheConfigCacheInsertData(const std::string& value); /* engine config */ - Status CheckEngineConfigUseBlasThreshold(const std::string &value); - Status CheckEngineConfigOmpThreadNum(const std::string &value); + Status + CheckEngineConfigUseBlasThreshold(const std::string& value); + Status + CheckEngineConfigOmpThreadNum(const std::string& value); /* resource config */ - Status CheckResourceConfigMode(const std::string &value); - Status CheckResourceConfigPool(const std::vector &value); + Status + CheckResourceConfigMode(const std::string& value); + Status + CheckResourceConfigPool(const std::vector& value); /////////////////////////////////////////////////////////////////////////// /* server config */ - std::string GetServerConfigStrAddress(); - std::string GetServerConfigStrPort(); - std::string GetServerConfigStrDeployMode(); - std::string GetServerConfigStrTimeZone(); + std::string + GetServerConfigStrAddress(); + std::string + GetServerConfigStrPort(); + std::string + GetServerConfigStrDeployMode(); + std::string + GetServerConfigStrTimeZone(); /* db config */ - std::string GetDBConfigStrPrimaryPath(); - std::string GetDBConfigStrSecondaryPath(); - std::string GetDBConfigStrBackendUrl(); - std::string GetDBConfigStrArchiveDiskThreshold(); - std::string GetDBConfigStrArchiveDaysThreshold(); - std::string GetDBConfigStrInsertBufferSize(); - std::string GetDBConfigStrBuildIndexGPU(); + std::string + GetDBConfigStrPrimaryPath(); + std::string + GetDBConfigStrSecondaryPath(); + std::string + GetDBConfigStrBackendUrl(); + std::string + GetDBConfigStrArchiveDiskThreshold(); + std::string + GetDBConfigStrArchiveDaysThreshold(); + std::string + GetDBConfigStrInsertBufferSize(); + std::string + GetDBConfigStrBuildIndexGPU(); /* metric config */ - std::string GetMetricConfigStrEnableMonitor(); - std::string GetMetricConfigStrCollector(); - std::string GetMetricConfigStrPrometheusPort(); + std::string + GetMetricConfigStrEnableMonitor(); + std::string + GetMetricConfigStrCollector(); + std::string + GetMetricConfigStrPrometheusPort(); /* cache config */ - std::string GetCacheConfigStrCpuCacheCapacity(); - std::string GetCacheConfigStrCpuCacheThreshold(); - std::string GetCacheConfigStrGpuCacheCapacity(); - std::string GetCacheConfigStrGpuCacheThreshold(); - std::string GetCacheConfigStrCacheInsertData(); + std::string + GetCacheConfigStrCpuCacheCapacity(); + std::string + GetCacheConfigStrCpuCacheThreshold(); + std::string + GetCacheConfigStrGpuCacheCapacity(); + std::string + GetCacheConfigStrGpuCacheThreshold(); + std::string + GetCacheConfigStrCacheInsertData(); /* engine config */ - std::string GetEngineConfigStrUseBlasThreshold(); - std::string GetEngineConfigStrOmpThreadNum(); + std::string + GetEngineConfigStrUseBlasThreshold(); + std::string + GetEngineConfigStrOmpThreadNum(); /* resource config */ - std::string GetResourceConfigStrMode(); + std::string + GetResourceConfigStrMode(); public: /* server config */ - Status GetServerConfigAddress(std::string &value); - Status GetServerConfigPort(std::string &value); - Status GetServerConfigDeployMode(std::string &value); - Status GetServerConfigTimeZone(std::string &value); + Status + GetServerConfigAddress(std::string& value); + Status + GetServerConfigPort(std::string& value); + Status + GetServerConfigDeployMode(std::string& value); + Status + GetServerConfigTimeZone(std::string& value); /* db config */ - Status GetDBConfigPrimaryPath(std::string &value); - Status GetDBConfigSecondaryPath(std::string &value); - Status GetDBConfigBackendUrl(std::string &value); - Status GetDBConfigArchiveDiskThreshold(int32_t &value); - Status GetDBConfigArchiveDaysThreshold(int32_t &value); - Status GetDBConfigInsertBufferSize(int32_t &value); - Status GetDBConfigBuildIndexGPU(int32_t &value); + Status + GetDBConfigPrimaryPath(std::string& value); + Status + GetDBConfigSecondaryPath(std::string& value); + Status + GetDBConfigBackendUrl(std::string& value); + Status + GetDBConfigArchiveDiskThreshold(int32_t& value); + Status + GetDBConfigArchiveDaysThreshold(int32_t& value); + Status + GetDBConfigInsertBufferSize(int32_t& value); + Status + GetDBConfigBuildIndexGPU(int32_t& value); /* metric config */ - Status GetMetricConfigEnableMonitor(bool &value); - Status GetMetricConfigCollector(std::string &value); - Status GetMetricConfigPrometheusPort(std::string &value); + Status + GetMetricConfigEnableMonitor(bool& value); + Status + GetMetricConfigCollector(std::string& value); + Status + GetMetricConfigPrometheusPort(std::string& value); /* cache config */ - Status GetCacheConfigCpuCacheCapacity(int32_t &value); - Status GetCacheConfigCpuCacheThreshold(float &value); - Status GetCacheConfigGpuCacheCapacity(int32_t &value); - Status GetCacheConfigGpuCacheThreshold(float &value); - Status GetCacheConfigCacheInsertData(bool &value); + Status + GetCacheConfigCpuCacheCapacity(int32_t& value); + Status + GetCacheConfigCpuCacheThreshold(float& value); + Status + GetCacheConfigGpuCacheCapacity(int32_t& value); + Status + GetCacheConfigGpuCacheThreshold(float& value); + Status + GetCacheConfigCacheInsertData(bool& value); /* engine config */ - Status GetEngineConfigUseBlasThreshold(int32_t &value); - Status GetEngineConfigOmpThreadNum(int32_t &value); + Status + GetEngineConfigUseBlasThreshold(int32_t& value); + Status + GetEngineConfigOmpThreadNum(int32_t& value); /* resource config */ - Status GetResourceConfigMode(std::string &value); - Status GetResourceConfigPool(std::vector &value); + Status + GetResourceConfigMode(std::string& value); + Status + GetResourceConfigPool(std::vector& value); public: /* server config */ - Status SetServerConfigAddress(const std::string &value); - Status SetServerConfigPort(const std::string &value); - Status SetServerConfigDeployMode(const std::string &value); - Status SetServerConfigTimeZone(const std::string &value); + Status + SetServerConfigAddress(const std::string& value); + Status + SetServerConfigPort(const std::string& value); + Status + SetServerConfigDeployMode(const std::string& value); + Status + SetServerConfigTimeZone(const std::string& value); /* db config */ - Status SetDBConfigPrimaryPath(const std::string &value); - Status SetDBConfigSecondaryPath(const std::string &value); - Status SetDBConfigBackendUrl(const std::string &value); - Status SetDBConfigArchiveDiskThreshold(const std::string &value); - Status SetDBConfigArchiveDaysThreshold(const std::string &value); - Status SetDBConfigInsertBufferSize(const std::string &value); - Status SetDBConfigBuildIndexGPU(const std::string &value); + Status + SetDBConfigPrimaryPath(const std::string& value); + Status + SetDBConfigSecondaryPath(const std::string& value); + Status + SetDBConfigBackendUrl(const std::string& value); + Status + SetDBConfigArchiveDiskThreshold(const std::string& value); + Status + SetDBConfigArchiveDaysThreshold(const std::string& value); + Status + SetDBConfigInsertBufferSize(const std::string& value); + Status + SetDBConfigBuildIndexGPU(const std::string& value); /* metric config */ - Status SetMetricConfigEnableMonitor(const std::string &value); - Status SetMetricConfigCollector(const std::string &value); - Status SetMetricConfigPrometheusPort(const std::string &value); + Status + SetMetricConfigEnableMonitor(const std::string& value); + Status + SetMetricConfigCollector(const std::string& value); + Status + SetMetricConfigPrometheusPort(const std::string& value); /* cache config */ - Status SetCacheConfigCpuCacheCapacity(const std::string &value); - Status SetCacheConfigCpuCacheThreshold(const std::string &value); - Status SetCacheConfigGpuCacheCapacity(const std::string &value); - Status SetCacheConfigGpuCacheThreshold(const std::string &value); - Status SetCacheConfigCacheInsertData(const std::string &value); + Status + SetCacheConfigCpuCacheCapacity(const std::string& value); + Status + SetCacheConfigCpuCacheThreshold(const std::string& value); + Status + SetCacheConfigGpuCacheCapacity(const std::string& value); + Status + SetCacheConfigGpuCacheThreshold(const std::string& value); + Status + SetCacheConfigCacheInsertData(const std::string& value); /* engine config */ - Status SetEngineConfigUseBlasThreshold(const std::string &value); - Status SetEngineConfigOmpThreadNum(const std::string &value); + Status + SetEngineConfigUseBlasThreshold(const std::string& value); + Status + SetEngineConfigOmpThreadNum(const std::string& value); /* resource config */ - Status SetResourceConfigMode(const std::string &value); + Status + SetResourceConfigMode(const std::string& value); private: std::unordered_map> config_map_; std::mutex mutex_; }; -} // namespace server -} // namespace milvus -} // namespace zilliz - +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index d764570966..c0e02f24e4 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "server/DBWrapper.h" #include "Config.h" #include "db/DBFactory.h" @@ -23,9 +22,9 @@ #include "utils/Log.h" #include "utils/StringHelpFunctions.h" -#include -#include #include +#include +#include namespace zilliz { namespace milvus { @@ -34,10 +33,11 @@ namespace server { DBWrapper::DBWrapper() { } -Status DBWrapper::StartService() { +Status +DBWrapper::StartService() { Config& config = Config::GetInstance(); Status s; - //db config + // db config engine::DBOptions opt; s = config.GetDBConfigBackendUrl(opt.meta_.backend_uri_); @@ -70,9 +70,8 @@ Status DBWrapper::StartService() { } else if (mode == "cluster_writable") { opt.mode_ = engine::DBOptions::MODE::CLUSTER_WRITABLE; } else { - std::cerr << - "ERROR: mode specified in server_config must be ['single', 'cluster_readonly', 'cluster_writable']" - << std::endl; + std::cerr << "ERROR: mode specified in server_config must be ['single', 'cluster_readonly', 'cluster_writable']" + << std::endl; kill(0, SIGUSR1); } @@ -86,18 +85,18 @@ Status DBWrapper::StartService() { } else { uint32_t sys_thread_cnt = 8; if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { - omp_thread = (int32_t)ceil(sys_thread_cnt*0.5); + omp_thread = (int32_t)ceil(sys_thread_cnt * 0.5); omp_set_num_threads(omp_thread); } } - //init faiss global variable + // init faiss global variable int32_t use_blas_threshold; s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold); if (!s.ok()) return s; faiss::distance_compute_blas_threshold = use_blas_threshold; - //set archive config + // set archive config engine::ArchiveConf::CriteriaT criterial; int32_t disk, days; s = config.GetDBConfigArchiveDiskThreshold(disk); @@ -113,7 +112,7 @@ Status DBWrapper::StartService() { } opt.meta_.archive_conf_.SetCriterias(criterial); - //create db root folder + // create db root folder Status status = CommonUtil::CreateDirectory(opt.meta_.path_); if (!status.ok()) { std::cerr << "ERROR! Failed to create database root path: " << opt.meta_.path_ << std::endl; @@ -128,10 +127,10 @@ Status DBWrapper::StartService() { } } - //create db instance + // create db instance try { db_ = engine::DBFactory::Build(opt); - } catch(std::exception& ex) { + } catch (std::exception& ex) { std::cerr << "ERROR! Failed to open database: " << ex.what() << std::endl; kill(0, SIGUSR1); } @@ -141,7 +140,8 @@ Status DBWrapper::StartService() { return Status::OK(); } -Status DBWrapper::StopService() { +Status +DBWrapper::StopService() { if (db_) { db_->Stop(); } @@ -149,6 +149,6 @@ Status DBWrapper::StopService() { return Status::OK(); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/DBWrapper.h b/cpp/src/server/DBWrapper.h index 4648f4b59d..5a035e44bc 100644 --- a/cpp/src/server/DBWrapper.h +++ b/cpp/src/server/DBWrapper.h @@ -17,8 +17,8 @@ #pragma once -#include "utils/Status.h" #include "db/DB.h" +#include "utils/Status.h" #include @@ -32,19 +32,24 @@ class DBWrapper { ~DBWrapper() = default; public: - static DBWrapper &GetInstance() { + static DBWrapper& + GetInstance() { static DBWrapper wrapper; return wrapper; } - static engine::DBPtr DB() { + static engine::DBPtr + DB() { return GetInstance().EngineDB(); } - Status StartService(); - Status StopService(); + Status + StartService(); + Status + StopService(); - engine::DBPtr EngineDB() { + engine::DBPtr + EngineDB() { return db_; } @@ -52,6 +57,6 @@ class DBWrapper { engine::DBPtr db_; }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index f0675fc84d..b86f386acc 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -17,41 +17,39 @@ #include "server/Server.h" -#include #include #include #include #include +#include //#include -#include #include +#include -#include "server/grpc_impl/GrpcServer.h" +#include "DBWrapper.h" +#include "metrics/Metrics.h" +#include "scheduler/SchedInst.h" #include "server/Config.h" +#include "server/grpc_impl/GrpcServer.h" #include "utils/Log.h" #include "utils/LogUtil.h" #include "utils/SignalUtil.h" #include "utils/TimeRecorder.h" -#include "metrics/Metrics.h" -#include "scheduler/SchedInst.h" #include "wrapper/KnowhereResource.h" -#include "DBWrapper.h" namespace zilliz { namespace milvus { namespace server { -Server & +Server& Server::GetInstance() { static Server server; return server; } void -Server::Init(int64_t daemonized, - const std::string &pid_filename, - const std::string &config_filename, - const std::string &log_config_file) { +Server::Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename, + const std::string& log_config_file) { daemonized_ = daemonized; pid_filename_ = pid_filename; config_filename_ = config_filename; @@ -66,9 +64,9 @@ Server::Daemonize() { std::cout << "Milvus server run in daemonize mode"; -// std::string log_path(GetLogDirFullPath()); -// log_path += "zdb_server.(INFO/WARNNING/ERROR/CRITICAL)"; -// SERVER_LOG_INFO << "Log will be exported to: " + log_path); + // std::string log_path(GetLogDirFullPath()); + // log_path += "zdb_server.(INFO/WARNNING/ERROR/CRITICAL)"; + // SERVER_LOG_INFO << "Log will be exported to: " + log_path); pid_t pid = 0; @@ -160,7 +158,7 @@ Server::Start() { } /* log path is defined in Config file, so InitLog must be called after LoadConfig */ - Config &config = Config::GetInstance(); + Config& config = Config::GetInstance(); std::string time_zone; Status s = config.GetServerConfigTimeZone(time_zone); if (!s.ok()) { @@ -194,7 +192,7 @@ Server::Start() { StartService(); std::cout << "Milvus server start successfully." << std::endl; - } catch (std::exception &ex) { + } catch (std::exception& ex) { std::cerr << "Milvus server encounter exception: " << ex.what(); } } @@ -233,7 +231,7 @@ Server::Stop() { ErrorCode Server::LoadConfig() { - Config &config = Config::GetInstance(); + Config& config = Config::GetInstance(); Status s = config.LoadConfigFile(config_filename_); if (!s.ok()) { std::cerr << "Failed to load config file: " << config_filename_ << std::endl; @@ -264,6 +262,6 @@ Server::StopService() { engine::KnowhereResource::Finalize(); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index 9bd0ce13b5..b86945cef9 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -28,26 +28,32 @@ namespace server { class Server { public: - static Server &GetInstance(); + static Server& + GetInstance(); - void Init(int64_t daemonized, - const std::string &pid_filename, - const std::string &config_filename, - const std::string &log_config_file); + void + Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename, + const std::string& log_config_file); - void Start(); - void Stop(); + void + Start(); + void + Stop(); private: Server() = default; ~Server() = default; - void Daemonize(); + void + Daemonize(); - ErrorCode LoadConfig(); + ErrorCode + LoadConfig(); - void StartService(); - void StopService(); + void + StartService(); + void + StopService(); private: int64_t daemonized_ = 0; @@ -57,6 +63,6 @@ class Server { std::string log_config_file_; }; // Server -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp index a64422aab7..6089df6f8e 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "server/grpc_impl/GrpcRequestHandler.h" #include "server/grpc_impl/GrpcRequestTask.h" #include "utils/TimeRecorder.h" @@ -28,18 +27,16 @@ namespace server { namespace grpc { ::grpc::Status -GrpcRequestHandler::CreateTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableSchema *request, - ::milvus::grpc::Status *response) { +GrpcRequestHandler::CreateTable(::grpc::ServerContext* context, const ::milvus::grpc::TableSchema* request, + ::milvus::grpc::Status* response) { BaseTaskPtr task_ptr = CreateTableTask::Create(request); GrpcRequestScheduler::ExecTask(task_ptr, response); return ::grpc::Status::OK; } ::grpc::Status -GrpcRequestHandler::HasTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::BoolReply *response) { +GrpcRequestHandler::HasTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::BoolReply* response) { bool has_table = false; BaseTaskPtr task_ptr = HasTableTask::Create(request->table_name(), has_table); ::milvus::grpc::Status grpc_status; @@ -51,27 +48,24 @@ GrpcRequestHandler::HasTable(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::DropTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) { +GrpcRequestHandler::DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::Status* response) { BaseTaskPtr task_ptr = DropTableTask::Create(request->table_name()); GrpcRequestScheduler::ExecTask(task_ptr, response); return ::grpc::Status::OK; } ::grpc::Status -GrpcRequestHandler::CreateIndex(::grpc::ServerContext *context, - const ::milvus::grpc::IndexParam *request, - ::milvus::grpc::Status *response) { +GrpcRequestHandler::CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request, + ::milvus::grpc::Status* response) { BaseTaskPtr task_ptr = CreateIndexTask::Create(request); GrpcRequestScheduler::ExecTask(task_ptr, response); return ::grpc::Status::OK; } ::grpc::Status -GrpcRequestHandler::Insert(::grpc::ServerContext *context, - const ::milvus::grpc::InsertParam *request, - ::milvus::grpc::VectorIds *response) { +GrpcRequestHandler::Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request, + ::milvus::grpc::VectorIds* response) { BaseTaskPtr task_ptr = InsertTask::Create(request, response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -81,9 +75,8 @@ GrpcRequestHandler::Insert(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::Search(::grpc::ServerContext *context, - const ::milvus::grpc::SearchParam *request, - ::milvus::grpc::TopKQueryResultList *response) { +GrpcRequestHandler::Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, + ::milvus::grpc::TopKQueryResultList* response) { std::vector file_id_array; BaseTaskPtr task_ptr = SearchTask::Create(request, file_id_array, response); ::milvus::grpc::Status grpc_status; @@ -94,14 +87,13 @@ GrpcRequestHandler::Search(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::SearchInFiles(::grpc::ServerContext *context, - const ::milvus::grpc::SearchInFilesParam *request, - ::milvus::grpc::TopKQueryResultList *response) { +GrpcRequestHandler::SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, + ::milvus::grpc::TopKQueryResultList* response) { std::vector file_id_array; for (int i = 0; i < request->file_id_array_size(); i++) { file_id_array.push_back(request->file_id_array(i)); } - ::milvus::grpc::SearchInFilesParam *request_mutable = const_cast<::milvus::grpc::SearchInFilesParam *>(request); + ::milvus::grpc::SearchInFilesParam* request_mutable = const_cast<::milvus::grpc::SearchInFilesParam*>(request); BaseTaskPtr task_ptr = SearchTask::Create(request_mutable->mutable_search_param(), file_id_array, response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -111,9 +103,8 @@ GrpcRequestHandler::SearchInFiles(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::DescribeTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::TableSchema *response) { +GrpcRequestHandler::DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::TableSchema* response) { BaseTaskPtr task_ptr = DescribeTableTask::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -123,9 +114,8 @@ GrpcRequestHandler::DescribeTable(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::CountTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::TableRowCount *response) { +GrpcRequestHandler::CountTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::TableRowCount* response) { int64_t row_count = 0; BaseTaskPtr task_ptr = CountTableTask::Create(request->table_name(), row_count); ::milvus::grpc::Status grpc_status; @@ -137,9 +127,8 @@ GrpcRequestHandler::CountTable(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::ShowTables(::grpc::ServerContext *context, - const ::milvus::grpc::Command *request, - ::milvus::grpc::TableNameList *response) { +GrpcRequestHandler::ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, + ::milvus::grpc::TableNameList* response) { BaseTaskPtr task_ptr = ShowTablesTask::Create(response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -149,9 +138,8 @@ GrpcRequestHandler::ShowTables(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::Cmd(::grpc::ServerContext *context, - const ::milvus::grpc::Command *request, - ::milvus::grpc::StringReply *response) { +GrpcRequestHandler::Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, + ::milvus::grpc::StringReply* response) { std::string result; BaseTaskPtr task_ptr = CmdTask::Create(request->cmd(), result); ::milvus::grpc::Status grpc_status; @@ -163,9 +151,8 @@ GrpcRequestHandler::Cmd(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::DeleteByRange(::grpc::ServerContext *context, - const ::milvus::grpc::DeleteByRangeParam *request, - ::milvus::grpc::Status *response) { +GrpcRequestHandler::DeleteByRange(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByRangeParam* request, + ::milvus::grpc::Status* response) { BaseTaskPtr task_ptr = DeleteByRangeTask::Create(request); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -175,9 +162,8 @@ GrpcRequestHandler::DeleteByRange(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::PreloadTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) { +GrpcRequestHandler::PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::Status* response) { BaseTaskPtr task_ptr = PreloadTableTask::Create(request->table_name()); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -187,9 +173,8 @@ GrpcRequestHandler::PreloadTable(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::DescribeIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::IndexParam *response) { +GrpcRequestHandler::DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::IndexParam* response) { BaseTaskPtr task_ptr = DescribeIndexTask::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -199,9 +184,8 @@ GrpcRequestHandler::DescribeIndex(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::DropIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) { +GrpcRequestHandler::DropIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::Status* response) { BaseTaskPtr task_ptr = DropIndexTask::Create(request->table_name()); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -210,7 +194,7 @@ GrpcRequestHandler::DropIndex(::grpc::ServerContext *context, return ::grpc::Status::OK; } -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.h b/cpp/src/server/grpc_impl/GrpcRequestHandler.h index 549e1d9d35..73f05572de 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.h @@ -45,8 +45,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param context */ ::grpc::Status - CreateTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableSchema *request, ::milvus::grpc::Status *response) override; + CreateTable(::grpc::ServerContext* context, const ::milvus::grpc::TableSchema* request, + ::milvus::grpc::Status* response) override; /** * @brief Test table existence method @@ -64,8 +64,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param context */ ::grpc::Status - HasTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, ::milvus::grpc::BoolReply *response) override; + HasTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::BoolReply* response) override; /** * @brief Drop table method @@ -83,8 +83,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param context */ ::grpc::Status - DropTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, ::milvus::grpc::Status *response) override; + DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::Status* response) override; /** * @brief build index by table method @@ -102,8 +102,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param context */ ::grpc::Status - CreateIndex(::grpc::ServerContext *context, - const ::milvus::grpc::IndexParam *request, ::milvus::grpc::Status *response) override; + CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request, + ::milvus::grpc::Status* response) override; /** * @brief Insert vector array to table @@ -121,9 +121,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param response */ ::grpc::Status - Insert(::grpc::ServerContext *context, - const ::milvus::grpc::InsertParam *request, - ::milvus::grpc::VectorIds *response) override; + Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request, + ::milvus::grpc::VectorIds* response) override; /** * @brief Query vector @@ -146,9 +145,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param writer */ ::grpc::Status - Search(::grpc::ServerContext *context, - const ::milvus::grpc::SearchParam *request, - ::milvus::grpc::TopKQueryResultList *response) override; + Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, + ::milvus::grpc::TopKQueryResultList* response) override; /** * @brief Internal use query interface @@ -171,9 +169,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param writer */ ::grpc::Status - SearchInFiles(::grpc::ServerContext *context, - const ::milvus::grpc::SearchInFilesParam *request, - ::milvus::grpc::TopKQueryResultList *response) override; + SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, + ::milvus::grpc::TopKQueryResultList* response) override; /** * @brief Get table schema @@ -191,9 +188,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param response */ ::grpc::Status - DescribeTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::TableSchema *response) override; + DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::TableSchema* response) override; /** * @brief Get table row count @@ -211,9 +207,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param context */ ::grpc::Status - CountTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::TableRowCount *response) override; + CountTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::TableRowCount* response) override; /** * @brief List all tables in database @@ -231,9 +226,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param writer */ ::grpc::Status - ShowTables(::grpc::ServerContext *context, - const ::milvus::grpc::Command *request, - ::milvus::grpc::TableNameList *table_name_list) override; + ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, + ::milvus::grpc::TableNameList* table_name_list) override; /** * @brief Give the server status @@ -251,9 +245,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param response */ ::grpc::Status - Cmd(::grpc::ServerContext *context, - const ::milvus::grpc::Command *request, - ::milvus::grpc::StringReply *response) override; + Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, + ::milvus::grpc::StringReply* response) override; /** * @brief delete table by range @@ -270,9 +263,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param response */ ::grpc::Status - DeleteByRange(::grpc::ServerContext *context, - const ::milvus::grpc::DeleteByRangeParam *request, - ::milvus::grpc::Status *response) override; + DeleteByRange(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByRangeParam* request, + ::milvus::grpc::Status* response) override; /** * @brief preload table @@ -289,9 +281,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param response */ ::grpc::Status - PreloadTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) override; + PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::Status* response) override; /** * @brief Describe index @@ -308,9 +299,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param response */ ::grpc::Status - DescribeIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::IndexParam *response) override; + DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::IndexParam* response) override; /** * @brief Drop index @@ -327,13 +317,11 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param response */ ::grpc::Status - DropIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) override; + DropIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::Status* response) override; }; -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz - +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp index 4c58195f37..fc302414c2 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp @@ -70,13 +70,11 @@ ErrorMap(ErrorCode code) { return ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR; } } -} // namespace +} // namespace //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -GrpcBaseTask::GrpcBaseTask(const std::string &task_group, bool async) - : task_group_(task_group), - async_(async), - done_(false) { +GrpcBaseTask::GrpcBaseTask(const std::string& task_group, bool async) + : task_group_(task_group), async_(async), done_(false) { } GrpcBaseTask::~GrpcBaseTask() { @@ -97,7 +95,7 @@ GrpcBaseTask::Done() { } Status -GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string &error_msg) { +GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string& error_msg) { status_ = Status(error_code, error_msg); SERVER_LOG_ERROR << error_msg; return status_; @@ -106,16 +104,13 @@ GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string &error_msg) { Status GrpcBaseTask::WaitToFinish() { std::unique_lock lock(finish_mtx_); - finish_cond_.wait(lock, [this] { - return done_; - }); + finish_cond_.wait(lock, [this] { return done_; }); return status_; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -GrpcRequestScheduler::GrpcRequestScheduler() - : stopped_(false) { +GrpcRequestScheduler::GrpcRequestScheduler() : stopped_(false) { Start(); } @@ -124,17 +119,17 @@ GrpcRequestScheduler::~GrpcRequestScheduler() { } void -GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status) { +GrpcRequestScheduler::ExecTask(BaseTaskPtr& task_ptr, ::milvus::grpc::Status* grpc_status) { if (task_ptr == nullptr) { return; } - GrpcRequestScheduler &scheduler = GrpcRequestScheduler::GetInstance(); + GrpcRequestScheduler& scheduler = GrpcRequestScheduler::GetInstance(); scheduler.ExecuteTask(task_ptr); if (!task_ptr->IsAsync()) { task_ptr->WaitToFinish(); - const Status &status = task_ptr->status(); + const Status& status = task_ptr->status(); if (!status.ok()) { grpc_status->set_reason(status.message()); grpc_status->set_error_code(ErrorMap(status.code())); @@ -168,8 +163,7 @@ GrpcRequestScheduler::Stop() { } for (auto iter : execute_threads_) { - if (iter == nullptr) - continue; + if (iter == nullptr) continue; iter->join(); } @@ -178,7 +172,7 @@ GrpcRequestScheduler::Stop() { } Status -GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) { +GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr& task_ptr) { if (task_ptr == nullptr) { return Status::OK(); } @@ -190,10 +184,10 @@ GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) { } if (task_ptr->IsAsync()) { - return Status::OK(); //async execution, caller need to call WaitToFinish at somewhere + return Status::OK(); // async execution, caller need to call WaitToFinish at somewhere } - return task_ptr->WaitToFinish();//sync execution + return task_ptr->WaitToFinish(); // sync execution } void @@ -206,7 +200,7 @@ GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { BaseTaskPtr task = task_queue->Take(); if (task == nullptr) { SERVER_LOG_ERROR << "Take null from task queue, stop thread"; - break;//stop the thread + break; // stop the thread } try { @@ -214,14 +208,14 @@ GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { if (!status.ok()) { SERVER_LOG_ERROR << "Task failed with code: " << status.ToString(); } - } catch (std::exception &ex) { + } catch (std::exception& ex) { SERVER_LOG_ERROR << "Task failed to execute: " << ex.what(); } } } Status -GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { +GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr& task_ptr) { std::lock_guard lock(queue_mtx_); std::string group_name = task_ptr->TaskGroup(); @@ -232,7 +226,7 @@ GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { queue->Put(task_ptr); task_groups_.insert(std::make_pair(group_name, queue)); - //start a thread + // start a thread ThreadPtr thread = std::make_shared(&GrpcRequestScheduler::TakeTaskToExecute, this, queue); execute_threads_.push_back(thread); SERVER_LOG_INFO << "Create new thread for task group: " << group_name; @@ -241,7 +235,7 @@ GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { return Status::OK(); } -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h index df5357a4bb..b5a87c0bfa 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h @@ -17,16 +17,16 @@ #pragma once -#include "utils/Status.h" -#include "utils/BlockingQueue.h" #include "grpc/gen-status/status.grpc.pb.h" #include "grpc/gen-status/status.pb.h" +#include "utils/BlockingQueue.h" +#include "utils/Status.h" #include -#include -#include #include #include +#include +#include namespace zilliz { namespace milvus { @@ -35,33 +35,41 @@ namespace grpc { class GrpcBaseTask { protected: - explicit GrpcBaseTask(const std::string &task_group, bool async = false); + explicit GrpcBaseTask(const std::string& task_group, bool async = false); virtual ~GrpcBaseTask(); public: - Status Execute(); + Status + Execute(); - void Done(); + void + Done(); - Status WaitToFinish(); + Status + WaitToFinish(); - std::string TaskGroup() const { + std::string + TaskGroup() const { return task_group_; } - const Status &status() const { + const Status& + status() const { return status_; } - bool IsAsync() const { + bool + IsAsync() const { return async_; } protected: - virtual Status OnExecute() = 0; + virtual Status + OnExecute() = 0; - Status SetStatus(ErrorCode error_code, const std::string &msg); + Status + SetStatus(ErrorCode error_code, const std::string& msg); protected: mutable std::mutex finish_mtx_; @@ -80,27 +88,34 @@ using ThreadPtr = std::shared_ptr; class GrpcRequestScheduler { public: - static GrpcRequestScheduler &GetInstance() { + static GrpcRequestScheduler& + GetInstance() { static GrpcRequestScheduler scheduler; return scheduler; } - void Start(); + void + Start(); - void Stop(); + void + Stop(); - Status ExecuteTask(const BaseTaskPtr &task_ptr); + Status + ExecuteTask(const BaseTaskPtr& task_ptr); - static void ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status); + static void + ExecTask(BaseTaskPtr& task_ptr, ::milvus::grpc::Status* grpc_status); protected: GrpcRequestScheduler(); virtual ~GrpcRequestScheduler(); - void TakeTaskToExecute(TaskQueuePtr task_queue); + void + TakeTaskToExecute(TaskQueuePtr task_queue); - Status PutTaskToQueue(const BaseTaskPtr &task_ptr); + Status + PutTaskToQueue(const BaseTaskPtr& task_ptr); private: mutable std::mutex queue_mtx_; @@ -112,7 +127,7 @@ class GrpcRequestScheduler { bool stopped_; }; -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index 5702b80bec..7ced2e69c1 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -19,29 +19,29 @@ #include #include -#include #include +#include //#include -#include "server/Server.h" +#include "../../../version.h" +#include "GrpcServer.h" +#include "db/Utils.h" +#include "scheduler/SchedInst.h" #include "server/DBWrapper.h" +#include "server/Server.h" #include "utils/CommonUtil.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" #include "utils/ValidationUtil.h" -#include "GrpcServer.h" -#include "db/Utils.h" -#include "scheduler/SchedInst.h" -#include "../../../version.h" namespace zilliz { namespace milvus { namespace server { namespace grpc { -static const char *DQL_TASK_GROUP = "dql"; -static const char *DDL_DML_TASK_GROUP = "ddl_dml"; -static const char *PING_TASK_GROUP = "ping"; +static const char* DQL_TASK_GROUP = "dql"; +static const char* DDL_DML_TASK_GROUP = "ddl_dml"; +static const char* PING_TASK_GROUP = "ping"; using DB_META = zilliz::milvus::engine::meta::Meta; using DB_DATE = zilliz::milvus::engine::meta::DateT; @@ -82,10 +82,9 @@ IndexType(engine::EngineType type) { constexpr int64_t DAY_SECONDS = 24 * 60 * 60; Status -ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range> &range_array, - std::vector &dates) { +ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range>& range_array, std::vector& dates) { dates.clear(); - for (auto &range : range_array) { + for (auto& range : range_array) { time_t tt_start, tt_end; tm tm_start, tm_end; if (!CommonUtil::TimeStrToTime(range.start_value(), tt_start, tm_start)) { @@ -96,37 +95,34 @@ ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range> &range_array, return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); } - int64_t days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) / - DAY_SECONDS; + int64_t days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) / DAY_SECONDS; if (days == 0) { return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value() + " to " + range.end_value()); } - //range: [start_day, end_day) + // range: [start_day, end_day) for (int64_t i = 0; i < days; i++) { time_t tt_day = tt_start + DAY_SECONDS * i; tm tm_day; CommonUtil::ConvertTime(tt_day, tm_day); - int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + - tm_day.tm_mday;//according to db logic + int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + tm_day.tm_mday; // according to db logic dates.push_back(date); } } return Status::OK(); } -} // namespace +} // namespace //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CreateTableTask::CreateTableTask(const ::milvus::grpc::TableSchema *schema) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - schema_(schema) { +CreateTableTask::CreateTableTask(const ::milvus::grpc::TableSchema* schema) + : GrpcBaseTask(DDL_DML_TASK_GROUP), schema_(schema) { } BaseTaskPtr -CreateTableTask::Create(const ::milvus::grpc::TableSchema *schema) { +CreateTableTask::Create(const ::milvus::grpc::TableSchema* schema) { if (schema == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; @@ -139,7 +135,7 @@ CreateTableTask::OnExecute() { TimeRecorder rc("CreateTableTask"); try { - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(schema_->table_name()); if (!status.ok()) { return status; @@ -160,23 +156,23 @@ CreateTableTask::OnExecute() { return status; } - //step 2: construct table schema + // step 2: construct table schema engine::meta::TableSchema table_info; table_info.table_id_ = schema_->table_name(); - table_info.dimension_ = (uint16_t) schema_->dimension(); + table_info.dimension_ = (uint16_t)schema_->dimension(); table_info.index_file_size_ = schema_->index_file_size(); table_info.metric_type_ = schema_->metric_type(); - //step 3: create table + // step 3: create table status = DBWrapper::DB()->CreateTable(table_info); if (!status.ok()) { - //table could exist + // table could exist if (status.code() == DB_ALREADY_EXIST) { return Status(SERVER_INVALID_TABLE_NAME, status.message()); } return status; } - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -186,14 +182,12 @@ CreateTableTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DescribeTableTask::DescribeTableTask(const std::string &table_name, ::milvus::grpc::TableSchema *schema) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name), - schema_(schema) { +DescribeTableTask::DescribeTableTask(const std::string& table_name, ::milvus::grpc::TableSchema* schema) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name), schema_(schema) { } BaseTaskPtr -DescribeTableTask::Create(const std::string &table_name, ::milvus::grpc::TableSchema *schema) { +DescribeTableTask::Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema) { return std::shared_ptr(new DescribeTableTask(table_name, schema)); } @@ -202,13 +196,13 @@ DescribeTableTask::OnExecute() { TimeRecorder rc("DescribeTableTask"); try { - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; } - //step 2: get table info + // step 2: get table info engine::meta::TableSchema table_info; table_info.table_id_ = table_name_; status = DBWrapper::DB()->DescribeTable(table_info); @@ -220,7 +214,7 @@ DescribeTableTask::OnExecute() { schema_->set_dimension(table_info.dimension_); schema_->set_index_file_size(table_info.index_file_size_); schema_->set_metric_type(table_info.metric_type_); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -230,13 +224,12 @@ DescribeTableTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CreateIndexTask::CreateIndexTask(const ::milvus::grpc::IndexParam *index_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - index_param_(index_param) { +CreateIndexTask::CreateIndexTask(const ::milvus::grpc::IndexParam* index_param) + : GrpcBaseTask(DDL_DML_TASK_GROUP), index_param_(index_param) { } BaseTaskPtr -CreateIndexTask::Create(const ::milvus::grpc::IndexParam *index_param) { +CreateIndexTask::Create(const ::milvus::grpc::IndexParam* index_param) { if (index_param == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; @@ -249,7 +242,7 @@ CreateIndexTask::OnExecute() { try { TimeRecorder rc("CreateIndexTask"); - //step 1: check arguments + // step 1: check arguments std::string table_name_ = index_param_->table_name(); auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { @@ -266,7 +259,7 @@ CreateIndexTask::OnExecute() { return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); } - auto &grpc_index = index_param_->index(); + auto& grpc_index = index_param_->index(); status = ValidationUtil::ValidateTableIndexType(grpc_index.index_type()); if (!status.ok()) { return status; @@ -277,7 +270,7 @@ CreateIndexTask::OnExecute() { return status; } - //step 2: check table existence + // step 2: check table existence engine::TableIndex index; index.engine_type_ = grpc_index.index_type(); index.nlist_ = grpc_index.nlist(); @@ -287,7 +280,7 @@ CreateIndexTask::OnExecute() { } rc.ElapseFromBegin("totally cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -295,14 +288,12 @@ CreateIndexTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -HasTableTask::HasTableTask(const std::string &table_name, bool &has_table) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name), - has_table_(has_table) { +HasTableTask::HasTableTask(const std::string& table_name, bool& has_table) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name), has_table_(has_table) { } BaseTaskPtr -HasTableTask::Create(const std::string &table_name, bool &has_table) { +HasTableTask::Create(const std::string& table_name, bool& has_table) { return std::shared_ptr(new HasTableTask(table_name, has_table)); } @@ -311,20 +302,20 @@ HasTableTask::OnExecute() { try { TimeRecorder rc("HasTableTask"); - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; } - //step 2: check table existence + // step 2: check table existence status = DBWrapper::DB()->HasTable(table_name_, has_table_); if (!status.ok()) { return status; } rc.ElapseFromBegin("totally cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -332,13 +323,12 @@ HasTableTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DropTableTask::DropTableTask(const std::string &table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name) { +DropTableTask::DropTableTask(const std::string& table_name) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name) { } BaseTaskPtr -DropTableTask::Create(const std::string &table_name) { +DropTableTask::Create(const std::string& table_name) { return std::shared_ptr(new DropTableTask(table_name)); } @@ -347,13 +337,13 @@ DropTableTask::OnExecute() { try { TimeRecorder rc("DropTableTask"); - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; } - //step 2: check table existence + // step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = table_name_; status = DBWrapper::DB()->DescribeTable(table_info); @@ -367,7 +357,7 @@ DropTableTask::OnExecute() { rc.ElapseFromBegin("check validation"); - //step 3: Drop table + // step 3: Drop table std::vector dates; status = DBWrapper::DB()->DeleteTable(table_name_, dates); if (!status.ok()) { @@ -375,7 +365,7 @@ DropTableTask::OnExecute() { } rc.ElapseFromBegin("total cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -383,13 +373,12 @@ DropTableTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -ShowTablesTask::ShowTablesTask(::milvus::grpc::TableNameList *table_name_list) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_list_(table_name_list) { +ShowTablesTask::ShowTablesTask(::milvus::grpc::TableNameList* table_name_list) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_list_(table_name_list) { } BaseTaskPtr -ShowTablesTask::Create(::milvus::grpc::TableNameList *table_name_list) { +ShowTablesTask::Create(::milvus::grpc::TableNameList* table_name_list) { return std::shared_ptr(new ShowTablesTask(table_name_list)); } @@ -401,23 +390,19 @@ ShowTablesTask::OnExecute() { return statuts; } - for (auto &schema : schema_array) { + for (auto& schema : schema_array) { table_name_list_->add_table_names(schema.table_id_); } return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -InsertTask::InsertTask(const ::milvus::grpc::InsertParam *insert_param, - ::milvus::grpc::VectorIds *record_ids) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - insert_param_(insert_param), - record_ids_(record_ids) { +InsertTask::InsertTask(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) + : GrpcBaseTask(DDL_DML_TASK_GROUP), insert_param_(insert_param), record_ids_(record_ids) { } BaseTaskPtr -InsertTask::Create(const ::milvus::grpc::InsertParam *insert_param, - ::milvus::grpc::VectorIds *record_ids) { +InsertTask::Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) { if (insert_param == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; @@ -430,7 +415,7 @@ InsertTask::OnExecute() { try { TimeRecorder rc("InsertVectorTask"); - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(insert_param_->table_name()); if (!status.ok()) { return status; @@ -441,34 +426,32 @@ InsertTask::OnExecute() { if (!record_ids_->vector_id_array().empty()) { if (record_ids_->vector_id_array().size() != insert_param_->row_record_array_size()) { - return Status(SERVER_ILLEGAL_VECTOR_ID, - "Size of vector ids is not equal to row record array size"); + return Status(SERVER_ILLEGAL_VECTOR_ID, "Size of vector ids is not equal to row record array size"); } } - //step 2: check table existence + // step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = insert_param_->table_name(); status = DBWrapper::DB()->DescribeTable(table_info); if (!status.ok()) { if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, - "Table " + insert_param_->table_name() + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, "Table " + insert_param_->table_name() + " not exists"); } else { return status; } } - //step 3: check table flag - //all user provide id, or all internal id + // step 3: check table flag + // all user provide id, or all internal id bool user_provide_ids = !insert_param_->row_id_array().empty(); - //user already provided id before, all insert action require user id + // user already provided id before, all insert action require user id if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) && !user_provide_ids) { return Status(SERVER_ILLEGAL_VECTOR_ID, "Table vector ids are user defined, please provide id for this batch"); } - //user didn't provided id before, no need to provide user id + // user didn't provided id before, no need to provide user id if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) && user_provide_ids) { return Status(SERVER_ILLEGAL_VECTOR_ID, "Table vector ids are auto generated, no need to provide id for this batch"); @@ -477,12 +460,12 @@ InsertTask::OnExecute() { rc.RecordSection("check validation"); #ifdef MILVUS_ENABLE_PROFILING - std::string fname = "/tmp/insert_" + std::to_string(this->insert_param_->row_record_array_size()) - + ".profiling"; + std::string fname = + "/tmp/insert_" + std::to_string(this->insert_param_->row_record_array_size()) + ".profiling"; ProfilerStart(fname.c_str()); #endif - //step 4: prepare float data + // step 4: prepare float data std::vector vec_f(insert_param_->row_record_array_size() * table_info.dimension_, 0); // TODO: change to one dimension array in protobuf or use multiple-thread to copy the data @@ -493,25 +476,23 @@ InsertTask::OnExecute() { uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size(); if (vec_dim != table_info.dimension_) { ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; - std::string error_msg = "Invalid row record dimension: " + std::to_string(vec_dim) - + " vs. table dimension:" + - std::to_string(table_info.dimension_); + std::string error_msg = "Invalid row record dimension: " + std::to_string(vec_dim) + + " vs. table dimension:" + std::to_string(table_info.dimension_); return Status(error_code, error_msg); } - memcpy(&vec_f[i * table_info.dimension_], - insert_param_->row_record_array(i).vector_data().data(), + memcpy(&vec_f[i * table_info.dimension_], insert_param_->row_record_array(i).vector_data().data(), table_info.dimension_ * sizeof(float)); } rc.ElapseFromBegin("prepare vectors data"); - //step 5: insert vectors - auto vec_count = (uint64_t) insert_param_->row_record_array_size(); + // step 5: insert vectors + auto vec_count = (uint64_t)insert_param_->row_record_array_size(); std::vector vec_ids(insert_param_->row_id_array_size(), 0); if (!insert_param_->row_id_array().empty()) { - const int64_t *src_data = insert_param_->row_id_array().data(); - int64_t *target_data = vec_ids.data(); - memcpy(target_data, src_data, (size_t) (sizeof(int64_t) * insert_param_->row_id_array_size())); + const int64_t* src_data = insert_param_->row_id_array().data(); + int64_t* target_data = vec_ids.data(); + memcpy(target_data, src_data, (size_t)(sizeof(int64_t) * insert_param_->row_id_array_size())); } status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), vec_count, vec_f.data(), vec_ids); @@ -525,14 +506,14 @@ InsertTask::OnExecute() { auto ids_size = record_ids_->vector_id_array_size(); if (ids_size != vec_count) { - std::string msg = "Add " + std::to_string(vec_count) + " vectors but only return " - + std::to_string(ids_size) + " id"; + std::string msg = + "Add " + std::to_string(vec_count) + " vectors but only return " + std::to_string(ids_size) + " id"; return Status(SERVER_ILLEGAL_VECTOR_ID, msg); } - //step 6: update table flag - user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID - : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; + // step 6: update table flag + user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID : table_info.flag_ |= + engine::meta::FLAG_MASK_NO_USERID; status = DBWrapper::DB()->UpdateTableFlag(insert_param_->table_name(), table_info.flag_); #ifdef MILVUS_ENABLE_PROFILING @@ -541,7 +522,7 @@ InsertTask::OnExecute() { rc.RecordSection("add vectors to engine"); rc.ElapseFromBegin("total cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -549,9 +530,8 @@ InsertTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -SearchTask::SearchTask(const ::milvus::grpc::SearchParam *search_vector_infos, - const std::vector &file_id_array, - ::milvus::grpc::TopKQueryResultList *response) +SearchTask::SearchTask(const ::milvus::grpc::SearchParam* search_vector_infos, + const std::vector& file_id_array, ::milvus::grpc::TopKQueryResultList* response) : GrpcBaseTask(DQL_TASK_GROUP), search_param_(search_vector_infos), file_id_array_(file_id_array), @@ -559,9 +539,8 @@ SearchTask::SearchTask(const ::milvus::grpc::SearchParam *search_vector_infos, } BaseTaskPtr -SearchTask::Create(const ::milvus::grpc::SearchParam *search_vector_infos, - const std::vector &file_id_array, - ::milvus::grpc::TopKQueryResultList *response) { +SearchTask::Create(const ::milvus::grpc::SearchParam* search_vector_infos, + const std::vector& file_id_array, ::milvus::grpc::TopKQueryResultList* response) { if (search_vector_infos == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; @@ -578,14 +557,14 @@ SearchTask::OnExecute() { std::string hdr = "SearchTask(k=" + std::to_string(top_k) + ", nprob=" + std::to_string(nprobe) + ")"; TimeRecorder rc(hdr); - //step 1: check table name + // step 1: check table name std::string table_name_ = search_param_->table_name(); auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; } - //step 2: check table existence + // step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = table_name_; status = DBWrapper::DB()->DescribeTable(table_info); @@ -597,7 +576,7 @@ SearchTask::OnExecute() { } } - //step 3: check search parameter + // step 3: check search parameter status = ValidationUtil::ValidateSearchTopk(top_k, table_info); if (!status.ok()) { return status; @@ -612,7 +591,7 @@ SearchTask::OnExecute() { return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); } - //step 4: check date range, and convert to db dates + // step 4: check date range, and convert to db dates std::vector dates; std::vector<::milvus::grpc::Range> range_array; for (size_t i = 0; i < search_param_->query_range_array_size(); i++) { @@ -626,8 +605,7 @@ SearchTask::OnExecute() { rc.RecordSection("check validation"); - - //step 5: prepare float data + // step 5: prepare float data auto record_array_size = search_param_->query_record_array_size(); std::vector vec_f(record_array_size * table_info.dimension_, 0); for (size_t i = 0; i < record_array_size; i++) { @@ -637,33 +615,32 @@ SearchTask::OnExecute() { uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size(); if (query_vec_dim != table_info.dimension_) { ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; - std::string error_msg = "Invalid row record dimension: " + std::to_string(query_vec_dim) - + " vs. table dimension:" + std::to_string(table_info.dimension_); + std::string error_msg = "Invalid row record dimension: " + std::to_string(query_vec_dim) + + " vs. table dimension:" + std::to_string(table_info.dimension_); return Status(error_code, error_msg); } - memcpy(&vec_f[i * table_info.dimension_], - search_param_->query_record_array(i).vector_data().data(), + memcpy(&vec_f[i * table_info.dimension_], search_param_->query_record_array(i).vector_data().data(), table_info.dimension_ * sizeof(float)); } rc.RecordSection("prepare vector data"); - //step 6: search vectors + // step 6: search vectors engine::QueryResults results; - auto record_count = (uint64_t) search_param_->query_record_array().size(); + auto record_count = (uint64_t)search_param_->query_record_array().size(); #ifdef MILVUS_ENABLE_PROFILING - std::string fname = "/tmp/search_nq_" + std::to_string(this->search_param_->query_record_array_size()) - + ".profiling"; + std::string fname = + "/tmp/search_nq_" + std::to_string(this->search_param_->query_record_array_size()) + ".profiling"; ProfilerStart(fname.c_str()); #endif if (file_id_array_.empty()) { - status = DBWrapper::DB()->Query(table_name_, (size_t) top_k, record_count, nprobe, - vec_f.data(), dates, results); + status = + DBWrapper::DB()->Query(table_name_, (size_t)top_k, record_count, nprobe, vec_f.data(), dates, results); } else { - status = DBWrapper::DB()->Query(table_name_, file_id_array_, (size_t) top_k, - record_count, nprobe, vec_f.data(), dates, results); + status = DBWrapper::DB()->Query(table_name_, file_id_array_, (size_t)top_k, record_count, nprobe, + vec_f.data(), dates, results); } #ifdef MILVUS_ENABLE_PROFILING @@ -676,29 +653,29 @@ SearchTask::OnExecute() { } if (results.empty()) { - return Status::OK(); //empty table + return Status::OK(); // empty table } if (results.size() != record_count) { - std::string msg = "Search " + std::to_string(record_count) + " vectors but only return " - + std::to_string(results.size()) + " results"; + std::string msg = "Search " + std::to_string(record_count) + " vectors but only return " + + std::to_string(results.size()) + " results"; return Status(SERVER_ILLEGAL_SEARCH_RESULT, msg); } - //step 7: construct result array - for (auto &result : results) { - ::milvus::grpc::TopKQueryResult *topk_query_result = topk_result_list->add_topk_query_result(); - for (auto &pair : result) { - ::milvus::grpc::QueryResult *grpc_result = topk_query_result->add_query_result_arrays(); + // step 7: construct result array + for (auto& result : results) { + ::milvus::grpc::TopKQueryResult* topk_query_result = topk_result_list->add_topk_query_result(); + for (auto& pair : result) { + ::milvus::grpc::QueryResult* grpc_result = topk_query_result->add_query_result_arrays(); grpc_result->set_id(pair.first); grpc_result->set_distance(pair.second); } } - //step 8: print time cost percent + // step 8: print time cost percent rc.RecordSection("construct result and send"); rc.ElapseFromBegin("totally cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -706,14 +683,12 @@ SearchTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CountTableTask::CountTableTask(const std::string &table_name, int64_t &row_count) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name), - row_count_(row_count) { +CountTableTask::CountTableTask(const std::string& table_name, int64_t& row_count) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name), row_count_(row_count) { } BaseTaskPtr -CountTableTask::Create(const std::string &table_name, int64_t &row_count) { +CountTableTask::Create(const std::string& table_name, int64_t& row_count) { return std::shared_ptr(new CountTableTask(table_name, row_count)); } @@ -722,23 +697,23 @@ CountTableTask::OnExecute() { try { TimeRecorder rc("GetTableRowCountTask"); - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; } - //step 2: get row count + // step 2: get row count uint64_t row_count = 0; status = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); if (!status.ok()) { return status; } - row_count_ = (int64_t) row_count; + row_count_ = (int64_t)row_count; rc.ElapseFromBegin("total cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -746,14 +721,12 @@ CountTableTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CmdTask::CmdTask(const std::string &cmd, std::string &result) - : GrpcBaseTask(PING_TASK_GROUP), - cmd_(cmd), - result_(result) { +CmdTask::CmdTask(const std::string& cmd, std::string& result) + : GrpcBaseTask(PING_TASK_GROUP), cmd_(cmd), result_(result) { } BaseTaskPtr -CmdTask::Create(const std::string &cmd, std::string &result) { +CmdTask::Create(const std::string& cmd, std::string& result) { return std::shared_ptr(new CmdTask(cmd, result)); } @@ -771,13 +744,12 @@ CmdTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DeleteByRangeTask::DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - delete_by_range_param_(delete_by_range_param) { +DeleteByRangeTask::DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param) + : GrpcBaseTask(DDL_DML_TASK_GROUP), delete_by_range_param_(delete_by_range_param) { } BaseTaskPtr -DeleteByRangeTask::Create(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param) { +DeleteByRangeTask::Create(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param) { if (delete_by_range_param == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; @@ -791,14 +763,14 @@ DeleteByRangeTask::OnExecute() { try { TimeRecorder rc("DeleteByRangeTask"); - //step 1: check arguments + // step 1: check arguments std::string table_name = delete_by_range_param_->table_name(); auto status = ValidationUtil::ValidateTableName(table_name); if (!status.ok()) { return status; } - //step 2: check table existence + // step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = table_name; status = DBWrapper::DB()->DescribeTable(table_info); @@ -812,7 +784,7 @@ DeleteByRangeTask::OnExecute() { rc.ElapseFromBegin("check validation"); - //step 3: check date range, and convert to db dates + // step 3: check date range, and convert to db dates std::vector dates; ErrorCode error_code = SERVER_SUCCESS; std::string error_msg; @@ -832,7 +804,7 @@ DeleteByRangeTask::OnExecute() { if (!status.ok()) { return status; } - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -840,13 +812,12 @@ DeleteByRangeTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -PreloadTableTask::PreloadTableTask(const std::string &table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name) { +PreloadTableTask::PreloadTableTask(const std::string& table_name) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name) { } BaseTaskPtr -PreloadTableTask::Create(const std::string &table_name) { +PreloadTableTask::Create(const std::string& table_name) { return std::shared_ptr(new PreloadTableTask(table_name)); } @@ -855,20 +826,20 @@ PreloadTableTask::OnExecute() { try { TimeRecorder rc("PreloadTableTask"); - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; } - //step 2: check table existence + // step 2: check table existence status = DBWrapper::DB()->PreloadTable(table_name_); if (!status.ok()) { return status; } rc.ElapseFromBegin("totally cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -876,16 +847,12 @@ PreloadTableTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DescribeIndexTask::DescribeIndexTask(const std::string &table_name, - ::milvus::grpc::IndexParam *index_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name), - index_param_(index_param) { +DescribeIndexTask::DescribeIndexTask(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name), index_param_(index_param) { } BaseTaskPtr -DescribeIndexTask::Create(const std::string &table_name, - ::milvus::grpc::IndexParam *index_param) { +DescribeIndexTask::Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) { return std::shared_ptr(new DescribeIndexTask(table_name, index_param)); } @@ -894,13 +861,13 @@ DescribeIndexTask::OnExecute() { try { TimeRecorder rc("DescribeIndexTask"); - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; } - //step 2: check table existence + // step 2: check table existence engine::TableIndex index; status = DBWrapper::DB()->DescribeIndex(table_name_, index); if (!status.ok()) { @@ -912,7 +879,7 @@ DescribeIndexTask::OnExecute() { index_param_->mutable_index()->set_nlist(index.nlist_); rc.ElapseFromBegin("totally cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -920,13 +887,12 @@ DescribeIndexTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DropIndexTask::DropIndexTask(const std::string &table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name) { +DropIndexTask::DropIndexTask(const std::string& table_name) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name) { } BaseTaskPtr -DropIndexTask::Create(const std::string &table_name) { +DropIndexTask::Create(const std::string& table_name) { return std::shared_ptr(new DropIndexTask(table_name)); } @@ -935,7 +901,7 @@ DropIndexTask::OnExecute() { try { TimeRecorder rc("DropIndexTask"); - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; @@ -951,21 +917,21 @@ DropIndexTask::OnExecute() { return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); } - //step 2: check table existence + // step 2: check table existence status = DBWrapper::DB()->DropIndex(table_name_); if (!status.ok()) { return status; } rc.ElapseFromBegin("totally cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } return Status::OK(); } -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.h b/cpp/src/server/grpc_impl/GrpcRequestTask.h index 4c8c038d44..f0e77ea791 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.h +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.h @@ -17,17 +17,17 @@ #pragma once +#include "db/Types.h" #include "server/grpc_impl/GrpcRequestScheduler.h" #include "utils/Status.h" -#include "db/Types.h" #include "grpc/gen-milvus/milvus.grpc.pb.h" #include "grpc/gen-status/status.pb.h" #include #include -#include #include +#include namespace zilliz { namespace milvus { @@ -38,60 +38,60 @@ namespace grpc { class CreateTableTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const ::milvus::grpc::TableSchema *schema); + Create(const ::milvus::grpc::TableSchema* schema); protected: - explicit CreateTableTask(const ::milvus::grpc::TableSchema *request); + explicit CreateTableTask(const ::milvus::grpc::TableSchema* request); Status OnExecute() override; private: - const ::milvus::grpc::TableSchema *schema_; + const ::milvus::grpc::TableSchema* schema_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class HasTableTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &table_name, bool &has_table); + Create(const std::string& table_name, bool& has_table); protected: - HasTableTask(const std::string &request, bool &has_table); + HasTableTask(const std::string& request, bool& has_table); Status OnExecute() override; private: std::string table_name_; - bool &has_table_; + bool& has_table_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DescribeTableTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &table_name, ::milvus::grpc::TableSchema *schema); + Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema); protected: - DescribeTableTask(const std::string &table_name, ::milvus::grpc::TableSchema *schema); + DescribeTableTask(const std::string& table_name, ::milvus::grpc::TableSchema* schema); Status OnExecute() override; private: std::string table_name_; - ::milvus::grpc::TableSchema *schema_; + ::milvus::grpc::TableSchema* schema_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DropTableTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &table_name); + Create(const std::string& table_name); protected: - explicit DropTableTask(const std::string &table_name); + explicit DropTableTask(const std::string& table_name); Status OnExecute() override; @@ -104,133 +104,129 @@ class DropTableTask : public GrpcBaseTask { class CreateIndexTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const ::milvus::grpc::IndexParam *index_Param); + Create(const ::milvus::grpc::IndexParam* index_Param); protected: - explicit CreateIndexTask(const ::milvus::grpc::IndexParam *index_Param); + explicit CreateIndexTask(const ::milvus::grpc::IndexParam* index_Param); Status OnExecute() override; private: - const ::milvus::grpc::IndexParam *index_param_; + const ::milvus::grpc::IndexParam* index_param_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class ShowTablesTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(::milvus::grpc::TableNameList *table_name_list); + Create(::milvus::grpc::TableNameList* table_name_list); protected: - explicit ShowTablesTask(::milvus::grpc::TableNameList *table_name_list); + explicit ShowTablesTask(::milvus::grpc::TableNameList* table_name_list); Status OnExecute() override; private: - ::milvus::grpc::TableNameList *table_name_list_; + ::milvus::grpc::TableNameList* table_name_list_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class InsertTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const ::milvus::grpc::InsertParam *insert_Param, - ::milvus::grpc::VectorIds *record_ids_); + Create(const ::milvus::grpc::InsertParam* insert_Param, ::milvus::grpc::VectorIds* record_ids_); protected: - InsertTask(const ::milvus::grpc::InsertParam *insert_Param, - ::milvus::grpc::VectorIds *record_ids_); + InsertTask(const ::milvus::grpc::InsertParam* insert_Param, ::milvus::grpc::VectorIds* record_ids_); Status OnExecute() override; private: - const ::milvus::grpc::InsertParam *insert_param_; - ::milvus::grpc::VectorIds *record_ids_; + const ::milvus::grpc::InsertParam* insert_param_; + ::milvus::grpc::VectorIds* record_ids_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class SearchTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const ::milvus::grpc::SearchParam *search_param, - const std::vector &file_id_array, - ::milvus::grpc::TopKQueryResultList *response); + Create(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, + ::milvus::grpc::TopKQueryResultList* response); protected: - SearchTask(const ::milvus::grpc::SearchParam *search_param, - const std::vector &file_id_array, - ::milvus::grpc::TopKQueryResultList *response); + SearchTask(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, + ::milvus::grpc::TopKQueryResultList* response); Status OnExecute() override; private: - const ::milvus::grpc::SearchParam *search_param_; + const ::milvus::grpc::SearchParam* search_param_; std::vector file_id_array_; - ::milvus::grpc::TopKQueryResultList *topk_result_list; + ::milvus::grpc::TopKQueryResultList* topk_result_list; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CountTableTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &table_name, int64_t &row_count); + Create(const std::string& table_name, int64_t& row_count); protected: - CountTableTask(const std::string &table_name, int64_t &row_count); + CountTableTask(const std::string& table_name, int64_t& row_count); Status OnExecute() override; private: std::string table_name_; - int64_t &row_count_; + int64_t& row_count_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CmdTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &cmd, std::string &result); + Create(const std::string& cmd, std::string& result); protected: - CmdTask(const std::string &cmd, std::string &result); + CmdTask(const std::string& cmd, std::string& result); Status OnExecute() override; private: std::string cmd_; - std::string &result_; + std::string& result_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DeleteByRangeTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param); + Create(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param); protected: - explicit DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param); + explicit DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param); Status OnExecute() override; private: - const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param_; + const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class PreloadTableTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &table_name); + Create(const std::string& table_name); protected: - explicit PreloadTableTask(const std::string &table_name); + explicit PreloadTableTask(const std::string& table_name); Status OnExecute() override; @@ -243,29 +239,27 @@ class PreloadTableTask : public GrpcBaseTask { class DescribeIndexTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &table_name, - ::milvus::grpc::IndexParam *index_param); + Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); protected: - DescribeIndexTask(const std::string &table_name, - ::milvus::grpc::IndexParam *index_param); + DescribeIndexTask(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); Status OnExecute() override; private: std::string table_name_; - ::milvus::grpc::IndexParam *index_param_; + ::milvus::grpc::IndexParam* index_param_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DropIndexTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &table_name); + Create(const std::string& table_name); protected: - explicit DropIndexTask(const std::string &table_name); + explicit DropIndexTask(const std::string& table_name); Status OnExecute() override; @@ -274,7 +268,7 @@ class DropIndexTask : public GrpcBaseTask { std::string table_name_; }; -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcServer.cpp b/cpp/src/server/grpc_impl/GrpcServer.cpp index 065271dd55..dd7895c679 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcServer.cpp @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include "grpc/gen-milvus/milvus.grpc.pb.h" #include "server/grpc_impl/GrpcServer.h" +#include "GrpcRequestHandler.h" +#include "grpc/gen-milvus/milvus.grpc.pb.h" #include "server/Config.h" #include "server/DBWrapper.h" #include "utils/Log.h" -#include "GrpcRequestHandler.h" #include #include @@ -43,14 +43,16 @@ namespace grpc { constexpr int64_t MESSAGE_SIZE = -1; -//this class is to check port occupation during server start +// this class is to check port occupation during server start class NoReusePortOption : public ::grpc::ServerBuilderOption { public: - void UpdateArguments(::grpc::ChannelArguments *args) override { + void + UpdateArguments(::grpc::ChannelArguments* args) override { args->SetInt(GRPC_ARG_ALLOW_REUSEPORT, 0); } - void UpdatePlugins(std::vector> *plugins) override { + void + UpdatePlugins(std::vector>* plugins) override { } }; @@ -70,7 +72,7 @@ GrpcServer::Stop() { Status GrpcServer::StartService() { - Config &config = Config::GetInstance(); + Config& config = Config::GetInstance(); std::string address, port; Status s; @@ -87,7 +89,7 @@ GrpcServer::StartService() { ::grpc::ServerBuilder builder; builder.SetOption(std::unique_ptr<::grpc::ServerBuilderOption>(new NoReusePortOption)); - builder.SetMaxReceiveMessageSize(MESSAGE_SIZE); //default 4 * 1024 * 1024 + builder.SetMaxReceiveMessageSize(MESSAGE_SIZE); // default 4 * 1024 * 1024 builder.SetMaxSendMessageSize(MESSAGE_SIZE); builder.SetCompressionAlgorithmSupportStatus(GRPC_COMPRESS_STREAM_GZIP, true); @@ -114,7 +116,7 @@ GrpcServer::StopService() { return Status::OK(); } -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcServer.h b/cpp/src/server/grpc_impl/GrpcServer.h index 9101f144b3..9f3f827936 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.h +++ b/cpp/src/server/grpc_impl/GrpcServer.h @@ -19,11 +19,11 @@ #include "utils/Status.h" -#include +#include #include +#include #include #include -#include namespace zilliz { namespace milvus { @@ -32,27 +32,32 @@ namespace grpc { class GrpcServer { public: - static GrpcServer &GetInstance() { + static GrpcServer& + GetInstance() { static GrpcServer grpc_server; return grpc_server; } - void Start(); - void Stop(); + void + Start(); + void + Stop(); private: GrpcServer() = default; ~GrpcServer() = default; - Status StartService(); - Status StopService(); + Status + StartService(); + Status + StopService(); private: std::unique_ptr<::grpc::Server> server_ptr_; std::shared_ptr thread_ptr_; }; -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/BlockingQueue.h b/cpp/src/utils/BlockingQueue.h index b98fe28ef8..422f170cd6 100644 --- a/cpp/src/utils/BlockingQueue.h +++ b/cpp/src/utils/BlockingQueue.h @@ -27,29 +27,37 @@ namespace zilliz { namespace milvus { namespace server { -template +template class BlockingQueue { public: BlockingQueue() : mtx(), full_(), empty_() { } - BlockingQueue(const BlockingQueue &rhs) = delete; + BlockingQueue(const BlockingQueue& rhs) = delete; - BlockingQueue &operator=(const BlockingQueue &rhs) = delete; + BlockingQueue& + operator=(const BlockingQueue& rhs) = delete; - void Put(const T &task); + void + Put(const T& task); - T Take(); + T + Take(); - T Front(); + T + Front(); - T Back(); + T + Back(); - size_t Size(); + size_t + Size(); - bool Empty(); + bool + Empty(); - void SetCapacity(const size_t capacity); + void + SetCapacity(const size_t capacity); private: mutable std::mutex mtx; @@ -59,8 +67,8 @@ class BlockingQueue { size_t capacity_ = 32; }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz #include "./BlockingQueue.inl" diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index 0116e321e0..760c54c4a7 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -18,15 +18,15 @@ #include "utils/CommonUtil.h" #include "utils/Log.h" -#include -#include -#include -#include -#include #include +#include #include -#include +#include +#include #include +#include +#include +#include #include "boost/filesystem.hpp" @@ -45,29 +45,28 @@ namespace server { namespace fs = boost::filesystem; bool -CommonUtil::GetSystemMemInfo(uint64_t &total_mem, uint64_t &free_mem) { +CommonUtil::GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem) { struct sysinfo info; int ret = sysinfo(&info); total_mem = info.totalram; free_mem = info.freeram; - return ret == 0;//succeed 0, failed -1 + return ret == 0; // succeed 0, failed -1 } bool -CommonUtil::GetSystemAvailableThreads(uint32_t &thread_count) { - //threadCnt = std::thread::hardware_concurrency(); +CommonUtil::GetSystemAvailableThreads(uint32_t& thread_count) { + // threadCnt = std::thread::hardware_concurrency(); thread_count = sysconf(_SC_NPROCESSORS_CONF); thread_count *= THREAD_MULTIPLY_CPU; - if (thread_count == 0) - thread_count = 8; + if (thread_count == 0) thread_count = 8; return true; } bool -CommonUtil::IsDirectoryExist(const std::string &path) { - DIR *dp = nullptr; +CommonUtil::IsDirectoryExist(const std::string& path) { + DIR* dp = nullptr; if ((dp = opendir(path.c_str())) == nullptr) { return false; } @@ -77,7 +76,7 @@ CommonUtil::IsDirectoryExist(const std::string &path) { } Status -CommonUtil::CreateDirectory(const std::string &path) { +CommonUtil::CreateDirectory(const std::string& path) { if (path.empty()) { return Status::OK(); } @@ -85,7 +84,7 @@ CommonUtil::CreateDirectory(const std::string &path) { struct stat directory_stat; int status = stat(path.c_str(), &directory_stat); if (status == 0) { - return Status::OK();//already exist + return Status::OK(); // already exist } fs::path fs_path(path); @@ -97,7 +96,7 @@ CommonUtil::CreateDirectory(const std::string &path) { status = stat(path.c_str(), &directory_stat); if (status == 0) { - return Status::OK();//already exist + return Status::OK(); // already exist } int makeOK = mkdir(path.c_str(), S_IRWXU | S_IRGRP | S_IROTH); @@ -110,17 +109,16 @@ CommonUtil::CreateDirectory(const std::string &path) { namespace { void -RemoveDirectory(const std::string &path) { - DIR *dir = nullptr; - struct dirent *dmsg; +RemoveDirectory(const std::string& path) { + DIR* dir = nullptr; + struct dirent* dmsg; const int32_t buf_size = 256; char file_name[buf_size]; std::string folder_name = path + "/%s"; if ((dir = opendir(path.c_str())) != nullptr) { while ((dmsg = readdir(dir)) != nullptr) { - if (strcmp(dmsg->d_name, ".") != 0 - && strcmp(dmsg->d_name, "..") != 0) { + if (strcmp(dmsg->d_name, ".") != 0 && strcmp(dmsg->d_name, "..") != 0) { snprintf(file_name, buf_size, folder_name.c_str(), dmsg->d_name); std::string tmp = file_name; if (tmp.find(".") == std::string::npos) { @@ -136,10 +134,10 @@ RemoveDirectory(const std::string &path) { } remove(path.c_str()); } -} // namespace +} // namespace Status -CommonUtil::DeleteDirectory(const std::string &path) { +CommonUtil::DeleteDirectory(const std::string& path) { if (path.empty()) { return Status::OK(); } @@ -155,17 +153,17 @@ CommonUtil::DeleteDirectory(const std::string &path) { } bool -CommonUtil::IsFileExist(const std::string &path) { +CommonUtil::IsFileExist(const std::string& path) { return (access(path.c_str(), F_OK) == 0); } uint64_t -CommonUtil::GetFileSize(const std::string &path) { +CommonUtil::GetFileSize(const std::string& path) { struct stat file_info; if (stat(path.c_str(), &file_info) < 0) { return 0; } else { - return (uint64_t) file_info.st_size; + return (uint64_t)file_info.st_size; } } @@ -195,21 +193,13 @@ CommonUtil::GetExePath() { } bool -CommonUtil::TimeStrToTime(const std::string &time_str, - time_t &time_integer, - tm &time_struct, - const std::string &format) { +CommonUtil::TimeStrToTime(const std::string& time_str, time_t& time_integer, tm& time_struct, + const std::string& format) { time_integer = 0; memset(&time_struct, 0, sizeof(tm)); - int ret = sscanf(time_str.c_str(), - format.c_str(), - &(time_struct.tm_year), - &(time_struct.tm_mon), - &(time_struct.tm_mday), - &(time_struct.tm_hour), - &(time_struct.tm_min), - &(time_struct.tm_sec)); + int ret = sscanf(time_str.c_str(), format.c_str(), &(time_struct.tm_year), &(time_struct.tm_mon), + &(time_struct.tm_mday), &(time_struct.tm_hour), &(time_struct.tm_min), &(time_struct.tm_sec)); if (ret <= 0) { return false; } @@ -222,15 +212,15 @@ CommonUtil::TimeStrToTime(const std::string &time_str, } void -CommonUtil::ConvertTime(time_t time_integer, tm &time_struct) { +CommonUtil::ConvertTime(time_t time_integer, tm& time_struct) { localtime_r(&time_integer, &time_struct); } void -CommonUtil::ConvertTime(tm time_struct, time_t &time_integer) { +CommonUtil::ConvertTime(tm time_struct, time_t& time_integer) { time_integer = mktime(&time_struct); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/CommonUtil.h b/cpp/src/utils/CommonUtil.h old mode 100755 new mode 100644 index b059067d50..58ac41ecb1 --- a/cpp/src/utils/CommonUtil.h +++ b/cpp/src/utils/CommonUtil.h @@ -19,8 +19,8 @@ #include "utils/Status.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -28,28 +28,37 @@ namespace server { class CommonUtil { public: - static bool GetSystemMemInfo(uint64_t &total_mem, uint64_t &free_mem); - static bool GetSystemAvailableThreads(uint32_t &thread_count); + static bool + GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem); + static bool + GetSystemAvailableThreads(uint32_t& thread_count); - static bool IsFileExist(const std::string &path); - static uint64_t GetFileSize(const std::string &path); - static bool IsDirectoryExist(const std::string &path); - static Status CreateDirectory(const std::string &path); - static Status DeleteDirectory(const std::string &path); + static bool + IsFileExist(const std::string& path); + static uint64_t + GetFileSize(const std::string& path); + static bool + IsDirectoryExist(const std::string& path); + static Status + CreateDirectory(const std::string& path); + static Status + DeleteDirectory(const std::string& path); - static std::string GetFileName(std::string filename); - static std::string GetExePath(); + static std::string + GetFileName(std::string filename); + static std::string + GetExePath(); - static bool TimeStrToTime(const std::string &time_str, - time_t &time_integer, - tm &time_struct, - const std::string &format = "%d-%d-%d %d:%d:%d"); + static bool + TimeStrToTime(const std::string& time_str, time_t& time_integer, tm& time_struct, + const std::string& format = "%d-%d-%d %d:%d:%d"); - static void ConvertTime(time_t time_integer, tm &time_struct); - static void ConvertTime(tm time_struct, time_t &time_integer); + static void + ConvertTime(time_t time_integer, tm& time_struct); + static void + ConvertTime(tm time_struct, time_t& time_integer); }; -} // namespace server -} // namespace milvus -} // namespace zilliz - +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index d0bcc5cf8d..59e9015afb 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -50,7 +50,7 @@ ToKnowhereErrorCode(const ErrorCode error_code) { return KNOWHERE_ERROR_CODE_BASE + error_code; } -//server error code +// server error code constexpr ErrorCode SERVER_UNEXPECTED_ERROR = ToServerErrorCode(1); constexpr ErrorCode SERVER_UNSUPPORTED_ERROR = ToServerErrorCode(2); constexpr ErrorCode SERVER_NULL_POINTER = ToServerErrorCode(3); @@ -83,7 +83,7 @@ constexpr ErrorCode SERVER_INVALID_INDEX_METRIC_TYPE = ToServerErrorCode(115); constexpr ErrorCode SERVER_INVALID_INDEX_FILE_SIZE = ToServerErrorCode(116); constexpr ErrorCode SERVER_OUT_OF_MEMORY = ToServerErrorCode(117); -//db error code +// db error code constexpr ErrorCode DB_META_TRANSACTION_FAILED = ToDbErrorCode(1); constexpr ErrorCode DB_ERROR = ToDbErrorCode(2); constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3); @@ -92,7 +92,7 @@ constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5); constexpr ErrorCode DB_INCOMPATIB_META = ToDbErrorCode(6); constexpr ErrorCode DB_INVALID_META_URI = ToDbErrorCode(7); -//knowhere error code +// knowhere error code constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1); constexpr ErrorCode KNOWHERE_INVALID_ARGUMENT = ToKnowhereErrorCode(2); constexpr ErrorCode KNOWHERE_UNEXPECTED_ERROR = ToKnowhereErrorCode(3); @@ -101,17 +101,18 @@ constexpr ErrorCode KNOWHERE_NO_SPACE = ToKnowhereErrorCode(4); namespace server { class ServerException : public std::exception { public: - ServerException(ErrorCode error_code, - const std::string &message = std::string()) + explicit ServerException(ErrorCode error_code, const std::string& message = std::string()) : error_code_(error_code), message_(message) { } public: - ErrorCode error_code() const { + ErrorCode + error_code() const { return error_code_; } - virtual const char *what() const noexcept { + virtual const char* + what() const noexcept { return message_.c_str(); } @@ -119,7 +120,7 @@ class ServerException : public std::exception { ErrorCode error_code_; std::string message_; }; -} // namespace server +} // namespace server } // namespace milvus } // namespace zilliz diff --git a/cpp/src/utils/Exception.h b/cpp/src/utils/Exception.h index 7e30c372bc..2ca61c4884 100644 --- a/cpp/src/utils/Exception.h +++ b/cpp/src/utils/Exception.h @@ -27,16 +27,16 @@ namespace milvus { class Exception : public std::exception { public: - Exception(ErrorCode code, const std::string &message) - : code_(code), - message_(message) { + Exception(ErrorCode code, const std::string& message) : code_(code), message_(message) { } - ErrorCode code() const throw() { + ErrorCode + code() const throw() { return code_; } - virtual const char *what() const throw() { + virtual const char* + what() const throw() { if (message_.empty()) { return "Default Exception."; } else { @@ -54,14 +54,12 @@ class Exception : public std::exception { class InvalidArgumentException : public Exception { public: - InvalidArgumentException() - : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") { + InvalidArgumentException() : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") { } - explicit InvalidArgumentException(const std::string &message) - : Exception(SERVER_INVALID_ARGUMENT, message) { + explicit InvalidArgumentException(const std::string& message) : Exception(SERVER_INVALID_ARGUMENT, message) { } }; -} // namespace milvus -} // namespace zilliz +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/Log.h b/cpp/src/utils/Log.h index b1402d9e3e..64a8622b73 100644 --- a/cpp/src/utils/Log.h +++ b/cpp/src/utils/Log.h @@ -52,5 +52,5 @@ namespace milvus { #define WRAPPER_LOG_ERROR LOG(ERROR) << WRAPPER_DOMAIN_NAME #define WRAPPER_LOG_FATAL LOG(FATAL) << WRAPPER_DOMAIN_NAME -} // namespace milvus -} // namespace zilliz +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index 0e710a6e3a..32e942bca3 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -18,8 +18,8 @@ #include "utils/LogUtil.h" #include -#include #include +#include namespace zilliz { namespace milvus { @@ -36,16 +36,16 @@ static int fatal_idx = 0; // TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename void -RolloutHandler(const char *filename, std::size_t size, el::Level level) { - char *dirc = strdup(filename); - char *basec = strdup(filename); - char *dir = dirname(dirc); - char *base = basename(basec); +RolloutHandler(const char* filename, std::size_t size, el::Level level) { + char* dirc = strdup(filename); + char* basec = strdup(filename); + char* dir = dirname(dirc); + char* base = basename(basec); std::string s(base); std::stringstream ss; - std::string - list[] = {"\\", " ", "\'", "\"", "*", "\?", "{", "}", ";", "<", ">", "|", "^", "&", "$", "#", "!", "`", "~"}; + std::string list[] = {"\\", " ", "\'", "\"", "*", "\?", "{", "}", ";", "<", + ">", "|", "^", "&", "$", "#", "!", "`", "~"}; std::string::size_type position; for (auto substr : list) { position = 0; @@ -82,7 +82,7 @@ RolloutHandler(const char *filename, std::size_t size, el::Level level) { } Status -InitLog(const std::string &log_config_file) { +InitLog(const std::string& log_config_file) { el::Configurations conf(log_config_file); el::Loggers::reconfigureAllLoggers(conf); @@ -93,6 +93,6 @@ InitLog(const std::string &log_config_file) { return Status::OK(); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/LogUtil.h b/cpp/src/utils/LogUtil.h index e3f7bed51f..eb396f266e 100644 --- a/cpp/src/utils/LogUtil.h +++ b/cpp/src/utils/LogUtil.h @@ -20,18 +20,18 @@ #include "utils/Status.h" #include "utils/easylogging++.h" -#include #include +#include namespace zilliz { namespace milvus { namespace server { Status -InitLog(const std::string &log_config_file); +InitLog(const std::string& log_config_file); void -RolloutHandler(const char *filename, std::size_t size, el::Level level); +RolloutHandler(const char* filename, std::size_t size, el::Level level); #define SHOW_LOCATION #ifdef SHOW_LOCATION @@ -40,6 +40,6 @@ RolloutHandler(const char *filename, std::size_t size, el::Level level); #define LOCATION_INFO "" #endif -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index 5f74852995..a01e7573c7 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -19,9 +19,9 @@ #include "src/server/Server.h" #include "utils/Log.h" -#include -#include #include +#include +#include namespace zilliz { namespace milvus { @@ -34,7 +34,7 @@ SignalUtil::HandleSignal(int signum) { case SIGUSR2: { SERVER_LOG_INFO << "Server received signal: " << signum; - server::Server &server = server::Server::GetInstance(); + server::Server& server = server::Server::GetInstance(); server.Stop(); exit(0); @@ -43,7 +43,7 @@ SignalUtil::HandleSignal(int signum) { SERVER_LOG_INFO << "Server received critical signal: " << signum; SignalUtil::PrintStacktrace(); - server::Server &server = server::Server::GetInstance(); + server::Server& server = server::Server::GetInstance(); server.Stop(); exit(1); @@ -56,9 +56,9 @@ SignalUtil::PrintStacktrace() { SERVER_LOG_INFO << "Call stack:"; const int size = 32; - void *array[size]; + void* array[size]; int stack_num = backtrace(array, size); - char **stacktrace = backtrace_symbols(array, stack_num); + char** stacktrace = backtrace_symbols(array, stack_num); for (int i = 0; i < stack_num; ++i) { std::string info = stacktrace[i]; SERVER_LOG_INFO << info; @@ -66,6 +66,6 @@ SignalUtil::PrintStacktrace() { free(stacktrace); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/SignalUtil.h b/cpp/src/utils/SignalUtil.h index c9cd41839b..414b320321 100644 --- a/cpp/src/utils/SignalUtil.h +++ b/cpp/src/utils/SignalUtil.h @@ -23,10 +23,12 @@ namespace server { class SignalUtil { public: - static void HandleSignal(int signum); - static void PrintStacktrace(); + static void + HandleSignal(int signum); + static void + PrintStacktrace(); }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/Status.cpp b/cpp/src/utils/Status.cpp index 5b512b3369..3f807a8a51 100644 --- a/cpp/src/utils/Status.cpp +++ b/cpp/src/utils/Status.cpp @@ -24,12 +24,12 @@ namespace milvus { constexpr int CODE_WIDTH = sizeof(StatusCode); -Status::Status(StatusCode code, const std::string &msg) { - //4 bytes store code - //4 bytes store message length - //the left bytes store message string - const uint32_t length = (uint32_t) msg.size(); - char *result = new char[length + sizeof(length) + CODE_WIDTH]; +Status::Status(StatusCode code, const std::string& msg) { + // 4 bytes store code + // 4 bytes store message length + // the left bytes store message string + const uint32_t length = (uint32_t)msg.size(); + char* result = new char[length + sizeof(length) + CODE_WIDTH]; std::memcpy(result, &code, CODE_WIDTH); std::memcpy(result + CODE_WIDTH, &length, sizeof(length)); memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length); @@ -37,38 +37,35 @@ Status::Status(StatusCode code, const std::string &msg) { state_ = result; } -Status::Status() - : state_(nullptr) { +Status::Status() : state_(nullptr) { } Status::~Status() { delete state_; } -Status::Status(const Status &s) - : state_(nullptr) { +Status::Status(const Status& s) : state_(nullptr) { CopyFrom(s); } -Status & -Status::operator=(const Status &s) { +Status& +Status::operator=(const Status& s) { CopyFrom(s); return *this; } -Status::Status(Status &&s) - : state_(nullptr) { +Status::Status(Status&& s) : state_(nullptr) { MoveFrom(s); } -Status & -Status::operator=(Status &&s) { +Status& +Status::operator=(Status&& s) { MoveFrom(s); return *this; } void -Status::CopyFrom(const Status &s) { +Status::CopyFrom(const Status& s) { delete state_; state_ = nullptr; if (s.state_ == nullptr) { @@ -79,11 +76,11 @@ Status::CopyFrom(const Status &s) { memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); int buff_len = length + sizeof(length) + CODE_WIDTH; state_ = new char[buff_len]; - memcpy((void *) state_, (void *) s.state_, buff_len); + memcpy((void*)state_, (void*)s.state_, buff_len); } void -Status::MoveFrom(Status &s) { +Status::MoveFrom(Status& s) { delete state_; state_ = s.state_; s.state_ = nullptr; @@ -113,19 +110,26 @@ Status::ToString() const { std::string result; switch (code()) { - case DB_SUCCESS:result = "OK "; + case DB_SUCCESS: + result = "OK "; break; - case DB_ERROR:result = "Error: "; + case DB_ERROR: + result = "Error: "; break; - case DB_META_TRANSACTION_FAILED:result = "Database error: "; + case DB_META_TRANSACTION_FAILED: + result = "Database error: "; break; - case DB_NOT_FOUND:result = "Not found: "; + case DB_NOT_FOUND: + result = "Not found: "; break; - case DB_ALREADY_EXIST:result = "Already exist: "; + case DB_ALREADY_EXIST: + result = "Already exist: "; break; - case DB_INVALID_PATH:result = "Invalid path: "; + case DB_INVALID_PATH: + result = "Invalid path: "; break; - default:result = "Error code(" + std::to_string(code()) + "): "; + default: + result = "Error code(" + std::to_string(code()) + "): "; break; } @@ -133,5 +137,5 @@ Status::ToString() const { return result; } -} // namespace milvus -} // namespace zilliz +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/Status.h b/cpp/src/utils/Status.h index 8f8f238979..4450a4c0e7 100644 --- a/cpp/src/utils/Status.h +++ b/cpp/src/utils/Status.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "utils/Error.h" @@ -29,19 +28,19 @@ using StatusCode = ErrorCode; class Status { public: - Status(StatusCode code, const std::string &msg); + Status(StatusCode code, const std::string& msg); Status(); ~Status(); - Status(const Status &s); + Status(const Status& s); - Status & - operator=(const Status &s); + Status& + operator=(const Status& s); - Status(Status &&s); + Status(Status&& s); - Status & - operator=(Status &&s); + Status& + operator=(Status&& s); static Status OK() { @@ -55,7 +54,7 @@ class Status { StatusCode code() const { - return (state_ == nullptr) ? 0 : *(StatusCode *) (state_); + return (state_ == nullptr) ? 0 : *(StatusCode*)(state_); } std::string @@ -66,14 +65,14 @@ class Status { private: inline void - CopyFrom(const Status &s); + CopyFrom(const Status& s); inline void - MoveFrom(Status &s); + MoveFrom(Status& s); private: - const char *state_ = nullptr; -}; // Status + const char* state_ = nullptr; +}; // Status -} // namespace milvus -} // namespace zilliz +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/StringHelpFunctions.cpp b/cpp/src/utils/StringHelpFunctions.cpp index 8c9e888d3a..444467845f 100644 --- a/cpp/src/utils/StringHelpFunctions.cpp +++ b/cpp/src/utils/StringHelpFunctions.cpp @@ -24,7 +24,7 @@ namespace milvus { namespace server { void -StringHelpFunctions::TrimStringBlank(std::string &string) { +StringHelpFunctions::TrimStringBlank(std::string& string) { if (!string.empty()) { static std::string s_format(" \n\r\t"); string.erase(0, string.find_first_not_of(s_format)); @@ -33,7 +33,7 @@ StringHelpFunctions::TrimStringBlank(std::string &string) { } void -StringHelpFunctions::TrimStringQuote(std::string &string, const std::string &qoute) { +StringHelpFunctions::TrimStringQuote(std::string& string, const std::string& qoute) { if (!string.empty()) { string.erase(0, string.find_first_not_of(qoute)); string.erase(string.find_last_not_of(qoute) + 1); @@ -41,9 +41,8 @@ StringHelpFunctions::TrimStringQuote(std::string &string, const std::string &qou } Status -StringHelpFunctions::SplitStringByDelimeter(const std::string &str, - const std::string &delimeter, - std::vector &result) { +StringHelpFunctions::SplitStringByDelimeter(const std::string& str, const std::string& delimeter, + std::vector& result) { if (str.empty()) { return Status::OK(); } @@ -64,10 +63,8 @@ StringHelpFunctions::SplitStringByDelimeter(const std::string &str, } Status -StringHelpFunctions::SplitStringByQuote(const std::string &str, - const std::string &delimeter, - const std::string "e, - std::vector &result) { +StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::string& delimeter, const std::string& quote, + std::vector& result) { if (quote.empty()) { return SplitStringByDelimeter(str, delimeter, result); } @@ -126,6 +123,6 @@ StringHelpFunctions::SplitStringByQuote(const std::string &str, return Status::OK(); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/StringHelpFunctions.h b/cpp/src/utils/StringHelpFunctions.h index ebffa075ad..06615fc740 100644 --- a/cpp/src/utils/StringHelpFunctions.h +++ b/cpp/src/utils/StringHelpFunctions.h @@ -19,8 +19,8 @@ #include "utils/Status.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -31,34 +31,34 @@ class StringHelpFunctions { StringHelpFunctions() = default; public: - static void TrimStringBlank(std::string &string); + static void + TrimStringBlank(std::string& string); - static void TrimStringQuote(std::string &string, const std::string &qoute); + static void + TrimStringQuote(std::string& string, const std::string& qoute); - //split string by delimeter ',' + // split string by delimeter ',' // a,b,c a | b | c // a,b, a | b | // ,b,c | b | c // ,b, | b | // ,, | | // a a - static Status SplitStringByDelimeter(const std::string &str, - const std::string &delimeter, - std::vector &result); + static Status + SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector& result); - //assume the table has two columns, quote='\"', delimeter=',' + // assume the table has two columns, quote='\"', delimeter=',' // a,b a | b // "aa,gg,yy",b aa,gg,yy | b // aa"dd,rr"kk,pp aadd,rrkk | pp // "aa,bb" aa,bb // 55,1122\"aa,bb\",yyy,\"kkk\" 55 | 1122aa,bb | yyy | kkk // "55,1122"aa,bb",yyy,"kkk" illegal - static Status SplitStringByQuote(const std::string &str, - const std::string &delimeter, - const std::string "e, - std::vector &result); + static Status + SplitStringByQuote(const std::string& str, const std::string& delimeter, const std::string& quote, + std::vector& result); }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/ThreadPool.h b/cpp/src/utils/ThreadPool.h index 0524f6206d..24c4d20ad5 100644 --- a/cpp/src/utils/ThreadPool.h +++ b/cpp/src/utils/ThreadPool.h @@ -17,18 +17,18 @@ #pragma once -#include -#include -#include -#include -#include #include -#include #include +#include +#include +#include +#include #include +#include #include +#include -#define MAX_THREADS_NUM 32 +#define MAX_THREADS_NUM 32 namespace zilliz { namespace milvus { @@ -37,9 +37,9 @@ class ThreadPool { public: explicit ThreadPool(size_t threads, size_t queue_size = 1000); - template - auto enqueue(F &&f, Args &&... args) - -> std::future::type>; + template + auto + enqueue(F&& f, Args&&... args) -> std::future::type>; ~ThreadPool(); @@ -61,37 +61,30 @@ class ThreadPool { }; // the constructor just launches some amount of workers -inline ThreadPool::ThreadPool(size_t threads, size_t queue_size) - : max_queue_size_(queue_size), stop(false) { +inline ThreadPool::ThreadPool(size_t threads, size_t queue_size) : max_queue_size_(queue_size), stop(false) { for (size_t i = 0; i < threads; ++i) - workers_.emplace_back( - [this] { - for (;;) { - std::function task; + workers_.emplace_back([this] { + for (;;) { + std::function task; - { - std::unique_lock lock(this->queue_mutex_); - this->condition_.wait(lock, - [this] { - return this->stop || !this->tasks_.empty(); - }); - if (this->stop && this->tasks_.empty()) - return; - task = std::move(this->tasks_.front()); - this->tasks_.pop(); - } - this->condition_.notify_all(); - - task(); + { + std::unique_lock lock(this->queue_mutex_); + this->condition_.wait(lock, [this] { return this->stop || !this->tasks_.empty(); }); + if (this->stop && this->tasks_.empty()) return; + task = std::move(this->tasks_.front()); + this->tasks_.pop(); } - }); + this->condition_.notify_all(); + + task(); + } + }); } // add new work item to the pool -template +template auto -ThreadPool::enqueue(F &&f, Args &&... args) --> std::future::type> { +ThreadPool::enqueue(F&& f, Args&&... args) -> std::future::type> { using return_type = typename std::result_of::type; auto task = std::make_shared >( @@ -100,17 +93,11 @@ ThreadPool::enqueue(F &&f, Args &&... args) std::future res = task->get_future(); { std::unique_lock lock(queue_mutex_); - this->condition_.wait(lock, - [this] { - return this->tasks_.size() < max_queue_size_; - }); + this->condition_.wait(lock, [this] { return this->tasks_.size() < max_queue_size_; }); // don't allow enqueueing after stopping the pool - if (stop) - throw std::runtime_error("enqueue on stopped ThreadPool"); + if (stop) throw std::runtime_error("enqueue on stopped ThreadPool"); - tasks_.emplace([task]() { - (*task)(); - }); + tasks_.emplace([task]() { (*task)(); }); } condition_.notify_all(); return res; @@ -123,11 +110,10 @@ inline ThreadPool::~ThreadPool() { stop = true; } condition_.notify_all(); - for (std::thread &worker : workers_) { + for (std::thread& worker : workers_) { worker.join(); } } -} // namespace milvus -} // namespace zilliz - +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/TimeRecorder.cpp b/cpp/src/utils/TimeRecorder.cpp index 5246b35f13..64a0c70fc9 100644 --- a/cpp/src/utils/TimeRecorder.cpp +++ b/cpp/src/utils/TimeRecorder.cpp @@ -21,10 +21,7 @@ namespace zilliz { namespace milvus { -TimeRecorder::TimeRecorder(const std::string &header, - int64_t log_level) : - header_(header), - log_level_(log_level) { +TimeRecorder::TimeRecorder(const std::string& header, int64_t log_level) : header_(header), log_level_(log_level) { start_ = last_ = stdclock::now(); } @@ -40,7 +37,7 @@ TimeRecorder::GetTimeSpanStr(double span) { } void -TimeRecorder::PrintTimeRecord(const std::string &msg, double span) { +TimeRecorder::PrintTimeRecord(const std::string& msg, double span) { std::string str_log; if (!header_.empty()) str_log += header_ + ": "; str_log += msg; @@ -81,7 +78,7 @@ TimeRecorder::PrintTimeRecord(const std::string &msg, double span) { } double -TimeRecorder::RecordSection(const std::string &msg) { +TimeRecorder::RecordSection(const std::string& msg) { stdclock::time_point curr = stdclock::now(); double span = (std::chrono::duration(curr - last_)).count(); last_ = curr; @@ -91,7 +88,7 @@ TimeRecorder::RecordSection(const std::string &msg) { } double -TimeRecorder::ElapseFromBegin(const std::string &msg) { +TimeRecorder::ElapseFromBegin(const std::string& msg) { stdclock::time_point curr = stdclock::now(); double span = (std::chrono::duration(curr - start_)).count(); @@ -99,5 +96,5 @@ TimeRecorder::ElapseFromBegin(const std::string &msg) { return span; } -} // namespace milvus -} // namespace zilliz +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/TimeRecorder.h b/cpp/src/utils/TimeRecorder.h index 2bb937e71f..8f795f01c3 100644 --- a/cpp/src/utils/TimeRecorder.h +++ b/cpp/src/utils/TimeRecorder.h @@ -17,8 +17,8 @@ #pragma once -#include #include +#include namespace zilliz { namespace milvus { @@ -27,19 +27,22 @@ class TimeRecorder { using stdclock = std::chrono::high_resolution_clock; public: - TimeRecorder(const std::string &header, - int64_t log_level = 1); + explicit TimeRecorder(const std::string& header, int64_t log_level = 1); - ~TimeRecorder();//trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5 + ~TimeRecorder(); // trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5 - double RecordSection(const std::string &msg); + double + RecordSection(const std::string& msg); - double ElapseFromBegin(const std::string &msg); + double + ElapseFromBegin(const std::string& msg); - static std::string GetTimeSpanStr(double span); + static std::string + GetTimeSpanStr(double span); private: - void PrintTimeRecord(const std::string &msg, double span); + void + PrintTimeRecord(const std::string& msg, double span); private: std::string header_; @@ -48,5 +51,5 @@ class TimeRecorder { int64_t log_level_; }; -} // namespace milvus -} // namespace zilliz +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index a6d83af1dc..7047e65d30 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -15,16 +15,15 @@ // specific language governing permissions and limitations // under the License. - #include "utils/ValidationUtil.h" -#include "db/engine/ExecutionEngine.h" #include "Log.h" +#include "db/engine/ExecutionEngine.h" -#include -#include #include -#include +#include #include +#include +#include namespace zilliz { namespace milvus { @@ -32,10 +31,10 @@ namespace server { constexpr size_t TABLE_NAME_SIZE_LIMIT = 255; constexpr int64_t TABLE_DIMENSION_LIMIT = 16384; -constexpr int32_t INDEX_FILE_SIZE_LIMIT = 4096; //index trigger size max = 4096 MB +constexpr int32_t INDEX_FILE_SIZE_LIMIT = 4096; // index trigger size max = 4096 MB Status -ValidationUtil::ValidateTableName(const std::string &table_name) { +ValidationUtil::ValidateTableName(const std::string& table_name) { // Table name shouldn't be empty. if (table_name.empty()) { std::string msg = "Empty table name"; @@ -84,8 +83,8 @@ ValidationUtil::ValidateTableDimension(int64_t dimension) { Status ValidationUtil::ValidateTableIndexType(int32_t index_type) { - int engine_type = (int) engine::EngineType(index_type); - if (engine_type <= 0 || engine_type > (int) engine::EngineType::MAX_VALUE) { + int engine_type = (int)engine::EngineType(index_type); + if (engine_type <= 0 || engine_type > (int)engine::EngineType::MAX_VALUE) { std::string msg = "Invalid index type: " + std::to_string(index_type); SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_TYPE, msg); @@ -118,7 +117,7 @@ ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { Status ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { - if (metric_type != (int32_t) engine::MetricType::L2 && metric_type != (int32_t) engine::MetricType::IP) { + if (metric_type != (int32_t)engine::MetricType::L2 && metric_type != (int32_t)engine::MetricType::IP) { std::string msg = "Invalid metric type: " + std::to_string(metric_type); SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_METRIC_TYPE, msg); @@ -127,7 +126,7 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { } Status -ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema &table_schema) { +ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) { if (top_k <= 0 || top_k > 2048) { std::string msg = "Invalid top k value: " + std::to_string(top_k); SERVER_LOG_ERROR << msg; @@ -138,7 +137,7 @@ ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchem } Status -ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema &table_schema) { +ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema) { if (nprobe <= 0 || nprobe > table_schema.nlist_) { std::string msg = "Invalid nprobe value: " + std::to_string(nprobe); SERVER_LOG_ERROR << msg; @@ -168,7 +167,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { } Status -ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t &memory) { +ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) { cudaDeviceProp deviceProp; auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index); if (cuda_err) { @@ -182,13 +181,14 @@ ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t &memory) { } Status -ValidationUtil::ValidateIpAddress(const std::string &ip_address) { +ValidationUtil::ValidateIpAddress(const std::string& ip_address) { struct in_addr address; int result = inet_pton(AF_INET, ip_address.c_str(), &address); switch (result) { - case 1:return Status::OK(); + case 1: + return Status::OK(); case 0: { std::string msg = "Invalid IP address: " + ip_address; SERVER_LOG_ERROR << msg; @@ -203,7 +203,7 @@ ValidationUtil::ValidateIpAddress(const std::string &ip_address) { } Status -ValidationUtil::ValidateStringIsNumber(const std::string &str) { +ValidationUtil::ValidateStringIsNumber(const std::string& str) { if (str.empty() || !std::all_of(str.begin(), str.end(), ::isdigit)) { return Status(SERVER_INVALID_ARGUMENT, "Invalid number"); } @@ -216,11 +216,10 @@ ValidationUtil::ValidateStringIsNumber(const std::string &str) { } Status -ValidationUtil::ValidateStringIsBool(const std::string &str) { +ValidationUtil::ValidateStringIsBool(const std::string& str) { std::string s = str; std::transform(s.begin(), s.end(), s.begin(), ::tolower); - if (s == "true" || s == "on" || s == "yes" || s == "1" || - s == "false" || s == "off" || s == "no" || s == "0" || + if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" || s.empty()) { return Status::OK(); } else { @@ -229,7 +228,7 @@ ValidationUtil::ValidateStringIsBool(const std::string &str) { } Status -ValidationUtil::ValidateStringIsFloat(const std::string &str) { +ValidationUtil::ValidateStringIsFloat(const std::string& str) { try { float val = std::stof(str); } catch (...) { @@ -239,19 +238,15 @@ ValidationUtil::ValidateStringIsFloat(const std::string &str) { } Status -ValidationUtil::ValidateDbURI(const std::string &uri) { +ValidationUtil::ValidateDbURI(const std::string& uri) { std::string dialectRegex = "(.*)"; std::string usernameRegex = "(.*)"; std::string passwordRegex = "(.*)"; std::string hostRegex = "(.*)"; std::string portRegex = "(.*)"; std::string dbNameRegex = "(.*)"; - std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + - usernameRegex + "\\:" + - passwordRegex + "\\@" + - hostRegex + "\\:" + - portRegex + "\\/" + - dbNameRegex; + std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + usernameRegex + "\\:" + passwordRegex + "\\@" + hostRegex + + "\\:" + portRegex + "\\/" + dbNameRegex; std::regex uriRegex(uriRegexStr); std::smatch pieces_match; @@ -265,17 +260,17 @@ ValidationUtil::ValidateDbURI(const std::string &uri) { okay = false; } -/* - * Could be DNS, skip checking - * - std::string host = pieces_match[4].str(); - if (!host.empty() && host != "localhost") { - if (ValidateIpAddress(host) != SERVER_SUCCESS) { - SERVER_LOG_ERROR << "Invalid host ip address in uri = " << host; - okay = false; - } - } -*/ + /* + * Could be DNS, skip checking + * + std::string host = pieces_match[4].str(); + if (!host.empty() && host != "localhost") { + if (ValidateIpAddress(host) != SERVER_SUCCESS) { + SERVER_LOG_ERROR << "Invalid host ip address in uri = " << host; + okay = false; + } + } + */ std::string port = pieces_match[5].str(); if (!port.empty()) { @@ -293,6 +288,6 @@ ValidationUtil::ValidateDbURI(const std::string &uri) { return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid db backend uri")); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index 44d6065a64..415347cec5 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "db/meta/MetaTypes.h" @@ -33,7 +32,7 @@ class ValidationUtil { public: static Status - ValidateTableName(const std::string &table_name); + ValidateTableName(const std::string& table_name); static Status ValidateTableDimension(int64_t dimension); @@ -51,33 +50,33 @@ class ValidationUtil { ValidateTableIndexMetricType(int32_t metric_type); static Status - ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema &table_schema); + ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema); static Status - ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema &table_schema); + ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema); static Status ValidateGpuIndex(uint32_t gpu_index); static Status - GetGpuMemory(uint32_t gpu_index, size_t &memory); + GetGpuMemory(uint32_t gpu_index, size_t& memory); static Status - ValidateIpAddress(const std::string &ip_address); + ValidateIpAddress(const std::string& ip_address); static Status - ValidateStringIsNumber(const std::string &str); + ValidateStringIsNumber(const std::string& str); static Status - ValidateStringIsBool(const std::string &str); + ValidateStringIsBool(const std::string& str); static Status - ValidateStringIsFloat(const std::string &str); + ValidateStringIsFloat(const std::string& str); static Status - ValidateDbURI(const std::string &uri); + ValidateDbURI(const std::string& uri); }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/ConfAdapter.cpp b/cpp/src/wrapper/ConfAdapter.cpp index ea6be1acf3..1ccbaee977 100644 --- a/cpp/src/wrapper/ConfAdapter.cpp +++ b/cpp/src/wrapper/ConfAdapter.cpp @@ -145,6 +145,10 @@ NSGConfAdapter::Match(const TempMetaConf &metaconf) { conf->out_degree = 50 + 5 * scale_factor; conf->candidate_pool_size = 200 + 100 * scale_factor; MatchBase(conf); + +// WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist +// << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d +// << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng; return conf; } diff --git a/cpp/src/wrapper/ConfAdapter.h b/cpp/src/wrapper/ConfAdapter.h index d0f032cb61..d8281d8887 100644 --- a/cpp/src/wrapper/ConfAdapter.h +++ b/cpp/src/wrapper/ConfAdapter.h @@ -47,6 +47,9 @@ class ConfAdapter { virtual knowhere::Config MatchSearch(const TempMetaConf &metaconf, const IndexType &type); +// virtual void +// Dump(){} + protected: static void MatchBase(knowhere::Config conf); }; diff --git a/cpp/src/wrapper/VecIndex.cpp b/cpp/src/wrapper/VecIndex.cpp index 734f66338f..ffc14ebd87 100644 --- a/cpp/src/wrapper/VecIndex.cpp +++ b/cpp/src/wrapper/VecIndex.cpp @@ -137,7 +137,7 @@ GetVecIndexFactory(const IndexType &type, const Config &cfg) { index = std::make_shared(gpu_device); break; } - case IndexType::NSG_MIX: { // TODO(linxj): bug. + case IndexType::NSG_MIX: { index = std::make_shared(gpu_device); break; }