admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-06 02:42:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/branch-0.5.0' into branch-0.5.0

Browse Source 
Former-commit-id: 206f6ac4f6c7af5c937aa88a35fd1156c0db9c72
This commit is contained in:
Yu Kun 2019-09-27 14:53:48 +08:00
commit a98cfa51c8
175 changed files with 3620 additions and 3271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
cpp/conf/server_config.template
View File

@ -1,39 +1,37 @@
# All the following configurations are default values.
server_config:
address: 0.0.0.0 # milvus server ip address (IPv4)
port: 19530 # port range: 1025 ~ 65534
mode: single # deployment type: single, cluster, read_only
address: 0.0.0.0 # milvus server ip address (IPv4)
port: 19530 # port range: 1025 ~ 65534
deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable
time_zone: UTC+8
db_config:
path: @MILVUS_DB_PATH@ # milvus database path
slave_path: # secondary database path, split by semicolon
primary_path: @MILVUS_DB_PATH@ # path used to store data and meta
secondary_path: # path used to store data only, split by semicolon
# URI format: dialect://username:password@host:port/database
# All parts except dialect are optional, but you MUST include the delimiters
# Currently dialect supports mysql or sqlite
backend_url: sqlite://:@:/
backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database
# Keep 'dialect://:@:/', and replace other texts with real values.
# Replace 'dialect' with 'mysql' or 'sqlite'
archive_disk_threshold: 0 # GB, file will be archived when disk usage exceed, 0 for no limit
archive_days_threshold: 0 # DAYS, older files will be archived, 0 for no limit
buffer_size: 4 # GB, maximum insert buffer size allowed
build_index_gpu: 0 # gpu id used for building index
insert_buffer_size: 4 # GB, maximum insert buffer size allowed
build_index_gpu: 0 # gpu id used for building index
metric_config:
auto_bootup: off # whether enable monitoring when bootup
collector: prometheus # prometheus
enable_monitor: false # enable monitoring or not
collector: prometheus # prometheus
prometheus_config:
port: 8080 # port prometheus used to fetch metrics
port: 8080 # port prometheus used to fetch metrics
cache_config:
cpu_mem_capacity: 16 # GB, CPU memory size used for cache
cpu_mem_threshold: 0.85 # percent of data kept when cache cleanup triggered
cache_insert_data: false # whether load data into cache when insert
cpu_mem_capacity: 16 # GB, CPU memory used for cache
cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered
cache_insert_data: false # whether load inserted data into cache
engine_config:
blas_threshold: 20
resource_config:
mode: simple
pool:
resource_pool:
- cpu
- gpu0

40
cpp/src/cache/Cache.h vendored
View File

@ -32,30 +32,40 @@ namespace cache {
template<typename ItemObj>
class Cache {
public:
public:
//mem_capacity, units:GB
Cache(int64_t capacity_gb, uint64_t cache_max_count);
~Cache() = default;
int64_t usage() const { return usage_; }
int64_t capacity() const { return capacity_; } //unit: BYTE
int64_t usage() const {
return usage_;
}
int64_t capacity() const {
return capacity_;
} //unit: BYTE
void set_capacity(int64_t capacity); //unit: BYTE
double freemem_percent() const { return freemem_percent_; };
void set_freemem_percent(double percent) { freemem_percent_ = percent; }
double freemem_percent() const {
return freemem_percent_;
}
void set_freemem_percent(double percent) {
freemem_percent_ = percent;
}
size_t size() const;
bool exists(const std::string& key);
ItemObj get(const std::string& key);
void insert(const std::string& key, const ItemObj& item);
void erase(const std::string& key);
bool exists(const std::string &key);
ItemObj get(const std::string &key);
void insert(const std::string &key, const ItemObj &item);
void erase(const std::string &key);
void print();
void clear();
private:
private:
void free_memory();
private:
private:
int64_t usage_;
int64_t capacity_;
double freemem_percent_;
@ -64,8 +74,8 @@ private:
mutable std::mutex mutex_;
};
} // cache
} // milvus
} // zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz
#include "cache/Cache.inl"
#include "cache/Cache.inl"

53
cpp/src/cache/Cache.inl vendored
View File

@ -33,29 +33,33 @@ Cache<ItemObj>::Cache(int64_t capacity, uint64_t cache_max_count)
}
template<typename ItemObj>
void Cache<ItemObj>::set_capacity(int64_t capacity) {
if(capacity > 0) {
void
Cache<ItemObj>::set_capacity(int64_t capacity) {
if (capacity > 0) {
capacity_ = capacity;
free_memory();
}
}
template<typename ItemObj>
size_t Cache<ItemObj>::size() const {
size_t
Cache<ItemObj>::size() const {
std::lock_guard<std::mutex> lock(mutex_);
return lru_.size();
}
template<typename ItemObj>
bool Cache<ItemObj>::exists(const std::string& key) {
bool
Cache<ItemObj>::exists(const std::string &key) {
std::lock_guard<std::mutex> lock(mutex_);
return lru_.exists(key);
}
template<typename ItemObj>
ItemObj Cache<ItemObj>::get(const std::string& key) {
ItemObj
Cache<ItemObj>::get(const std::string &key) {
std::lock_guard<std::mutex> lock(mutex_);
if(!lru_.exists(key)){
if (!lru_.exists(key)) {
return nullptr;
}
@ -63,8 +67,9 @@ ItemObj Cache<ItemObj>::get(const std::string& key) {
}
template<typename ItemObj>
void Cache<ItemObj>::insert(const std::string& key, const ItemObj& item) {
if(item == nullptr) {
void
Cache<ItemObj>::insert(const std::string &key, const ItemObj &item) {
if (item == nullptr) {
return;
}
@ -80,7 +85,7 @@ void Cache<ItemObj>::insert(const std::string& key, const ItemObj& item) {
//if key already exist, subtract old item size
if (lru_.exists(key)) {
const ItemObj& old_item = lru_.get(key);
const ItemObj &old_item = lru_.get(key);
usage_ -= old_item->size();
}
@ -107,13 +112,14 @@ void Cache<ItemObj>::insert(const std::string& key, const ItemObj& item) {
}
template<typename ItemObj>
void Cache<ItemObj>::erase(const std::string& key) {
void
Cache<ItemObj>::erase(const std::string &key) {
std::lock_guard<std::mutex> lock(mutex_);
if(!lru_.exists(key)){
if (!lru_.exists(key)) {
return;
}
const ItemObj& old_item = lru_.get(key);
const ItemObj &old_item = lru_.get(key);
usage_ -= old_item->size();
SERVER_LOG_DEBUG << "Erase " << key << " size: " << old_item->size();
@ -122,7 +128,8 @@ void Cache<ItemObj>::erase(const std::string& key) {
}
template<typename ItemObj>
void Cache<ItemObj>::clear() {
void
Cache<ItemObj>::clear() {
std::lock_guard<std::mutex> lock(mutex_);
lru_.clear();
usage_ = 0;
@ -131,12 +138,13 @@ void Cache<ItemObj>::clear() {
/* free memory space when CACHE occupation exceed its capacity */
template<typename ItemObj>
void Cache<ItemObj>::free_memory() {
void
Cache<ItemObj>::free_memory() {
if (usage_ <= capacity_) return;
int64_t threshhold = capacity_ * freemem_percent_;
int64_t delta_size = usage_ - threshhold;
if(delta_size <= 0) {
if (delta_size <= 0) {
delta_size = 1;//ensure at least one item erased
}
@ -148,8 +156,8 @@ void Cache<ItemObj>::free_memory() {
auto it = lru_.rbegin();
while (it != lru_.rend() && released_size < delta_size) {
auto& key = it->first;
auto& obj_ptr = it->second;
auto &key = it->first;
auto &obj_ptr = it->second;
key_array.emplace(key);
released_size += obj_ptr->size();
@ -159,7 +167,7 @@ void Cache<ItemObj>::free_memory() {
SERVER_LOG_DEBUG << "to be released memory size: " << released_size;
for (auto& key : key_array) {
for (auto &key : key_array) {
erase(key);
}
@ -167,7 +175,8 @@ void Cache<ItemObj>::free_memory() {
}
template<typename ItemObj>
void Cache<ItemObj>::print() {
void
Cache<ItemObj>::print() {
size_t cache_count = 0;
{
std::lock_guard<std::mutex> lock(mutex_);
@ -179,7 +188,7 @@ void Cache<ItemObj>::print() {
SERVER_LOG_DEBUG << "[Cache capacity]: " << capacity_ << " bytes";
}
} // cache
} // milvus
} // zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz

26
cpp/src/cache/CacheMgr.h vendored
View File

@ -22,22 +22,25 @@
#include "utils/Log.h"
#include "metrics/Metrics.h"
#include <string>
#include <memory>
namespace zilliz {
namespace milvus {
namespace cache {
template<typename ItemObj>
class CacheMgr {
public:
public:
virtual uint64_t ItemCount() const;
virtual bool ItemExists(const std::string& key);
virtual bool ItemExists(const std::string &key);
virtual ItemObj GetItem(const std::string& key);
virtual ItemObj GetItem(const std::string &key);
virtual void InsertItem(const std::string& key, const ItemObj& data);
virtual void InsertItem(const std::string &key, const ItemObj &data);
virtual void EraseItem(const std::string& key);
virtual void EraseItem(const std::string &key);
virtual void PrintInfo();
@ -47,18 +50,17 @@ public:
int64_t CacheCapacity() const;
void SetCapacity(int64_t capacity);
protected:
protected:
CacheMgr();
virtual ~CacheMgr();
protected:
protected:
using CachePtr = std::shared_ptr<Cache<ItemObj>>;
CachePtr cache_;
};
} // namespace cache
} // namespace milvus
} // namespace zilliz
}
}
}
#include "cache/CacheMgr.inl"
#include "cache/CacheMgr.inl"

58
cpp/src/cache/CacheMgr.inl vendored
View File

@ -30,18 +30,20 @@ CacheMgr<ItemObj>::~CacheMgr() {
}
template<typename ItemObj>
uint64_t CacheMgr<ItemObj>::ItemCount() const {
if(cache_ == nullptr) {
uint64_t
CacheMgr<ItemObj>::ItemCount() const {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return 0;
}
return (uint64_t)(cache_->size());
return (uint64_t) (cache_->size());
}
template<typename ItemObj>
bool CacheMgr<ItemObj>::ItemExists(const std::string& key) {
if(cache_ == nullptr) {
bool
CacheMgr<ItemObj>::ItemExists(const std::string &key) {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return false;
}
@ -50,8 +52,9 @@ bool CacheMgr<ItemObj>::ItemExists(const std::string& key) {
}
template<typename ItemObj>
ItemObj CacheMgr<ItemObj>::GetItem(const std::string& key) {
if(cache_ == nullptr) {
ItemObj
CacheMgr<ItemObj>::GetItem(const std::string &key) {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return nullptr;
}
@ -60,8 +63,9 @@ ItemObj CacheMgr<ItemObj>::GetItem(const std::string& key) {
}
template<typename ItemObj>
void CacheMgr<ItemObj>::InsertItem(const std::string& key, const ItemObj& data) {
if(cache_ == nullptr) {
void
CacheMgr<ItemObj>::InsertItem(const std::string &key, const ItemObj &data) {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return;
}
@ -71,8 +75,9 @@ void CacheMgr<ItemObj>::InsertItem(const std::string& key, const ItemObj& data)
}
template<typename ItemObj>
void CacheMgr<ItemObj>::EraseItem(const std::string& key) {
if(cache_ == nullptr) {
void
CacheMgr<ItemObj>::EraseItem(const std::string &key) {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return;
}
@ -82,8 +87,9 @@ void CacheMgr<ItemObj>::EraseItem(const std::string& key) {
}
template<typename ItemObj>
void CacheMgr<ItemObj>::PrintInfo() {
if(cache_ == nullptr) {
void
CacheMgr<ItemObj>::PrintInfo() {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return;
}
@ -92,8 +98,9 @@ void CacheMgr<ItemObj>::PrintInfo() {
}
template<typename ItemObj>
void CacheMgr<ItemObj>::ClearCache() {
if(cache_ == nullptr) {
void
CacheMgr<ItemObj>::ClearCache() {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return;
}
@ -102,8 +109,9 @@ void CacheMgr<ItemObj>::ClearCache() {
}
template<typename ItemObj>
int64_t CacheMgr<ItemObj>::CacheUsage() const {
if(cache_ == nullptr) {
int64_t
CacheMgr<ItemObj>::CacheUsage() const {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return 0;
}
@ -112,8 +120,9 @@ int64_t CacheMgr<ItemObj>::CacheUsage() const {
}
template<typename ItemObj>
int64_t CacheMgr<ItemObj>::CacheCapacity() const {
if(cache_ == nullptr) {
int64_t
CacheMgr<ItemObj>::CacheCapacity() const {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return 0;
}
@ -122,14 +131,15 @@ int64_t CacheMgr<ItemObj>::CacheCapacity() const {
}
template<typename ItemObj>
void CacheMgr<ItemObj>::SetCapacity(int64_t capacity) {
if(cache_ == nullptr) {
void
CacheMgr<ItemObj>::SetCapacity(int64_t capacity) {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return;
}
cache_->set_capacity(capacity);
}
}
}
}
} // namespace cache
} // namespace milvus
} // namespace zilliz

24
cpp/src/cache/CpuCacheMgr.cpp vendored
View File

@ -16,20 +16,22 @@
// under the License.
#include "CpuCacheMgr.h"
#include "cache/CpuCacheMgr.h"
#include "server/Config.h"
#include "utils/Log.h"
#include <utility>
namespace zilliz {
namespace milvus {
namespace cache {
namespace {
constexpr int64_t unit = 1024 * 1024 * 1024;
constexpr int64_t unit = 1024 * 1024 * 1024;
}
CpuCacheMgr::CpuCacheMgr() {
server::Config& config = server::Config::GetInstance();
server::Config &config = server::Config::GetInstance();
Status s;
int32_t cpu_mem_cap;
@ -38,7 +40,7 @@ CpuCacheMgr::CpuCacheMgr() {
SERVER_LOG_ERROR << s.message();
}
int64_t cap = cpu_mem_cap * unit;
cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL<<32);
cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL << 32);
float cpu_mem_threshold;
s = config.GetCacheConfigCpuMemThreshold(cpu_mem_threshold);
@ -53,20 +55,22 @@ CpuCacheMgr::CpuCacheMgr() {
}
}
CpuCacheMgr* CpuCacheMgr::GetInstance() {
CpuCacheMgr *
CpuCacheMgr::GetInstance() {
static CpuCacheMgr s_mgr;
return &s_mgr;
}
engine::VecIndexPtr CpuCacheMgr::GetIndex(const std::string& key) {
engine::VecIndexPtr
CpuCacheMgr::GetIndex(const std::string &key) {
DataObjPtr obj = GetItem(key);
if(obj != nullptr) {
if (obj != nullptr) {
return obj->data();
}
return nullptr;
}
}
}
}
} // namespace cache
} // namespace milvus
} // namespace zilliz

17
cpp/src/cache/CpuCacheMgr.h vendored
View File

@ -20,21 +20,24 @@
#include "CacheMgr.h"
#include "DataObj.h"
#include <string>
#include <memory>
namespace zilliz {
namespace milvus {
namespace cache {
class CpuCacheMgr : public CacheMgr<DataObjPtr> {
private:
private:
CpuCacheMgr();
public:
public:
//TODO: use smart pointer instead
static CpuCacheMgr* GetInstance();
static CpuCacheMgr *GetInstance();
engine::VecIndexPtr GetIndex(const std::string& key);
engine::VecIndexPtr GetIndex(const std::string &key);
};
}
}
}
} // namespace cache
} // namespace milvus
} // namespace zilliz

37
cpp/src/cache/DataObj.h vendored
View File

@ -27,38 +27,43 @@ namespace milvus {
namespace cache {
class DataObj {
public:
DataObj(const engine::VecIndexPtr& index)
: index_(index)
{}
public:
explicit DataObj(const engine::VecIndexPtr &index)
: index_(index) {
}
DataObj(const engine::VecIndexPtr& index, int64_t size)
: index_(index),
size_(size)
{}
DataObj(const engine::VecIndexPtr &index, int64_t size)
: index_(index),
size_(size) {
}
engine::VecIndexPtr data() { return index_; }
const engine::VecIndexPtr& data() const { return index_; }
engine::VecIndexPtr data() {
return index_;
}
const engine::VecIndexPtr &data() const {
return index_;
}
int64_t size() const {
if(index_ == nullptr) {
if (index_ == nullptr) {
return 0;
}
if(size_ > 0) {
if (size_ > 0) {
return size_;
}
return index_->Count() * index_->Dimension() * sizeof(float);
}
private:
private:
engine::VecIndexPtr index_ = nullptr;
int64_t size_ = 0;
};
using DataObjPtr = std::shared_ptr<DataObj>;
}
}
}
} // namespace cache
} // namespace milvus
} // namespace zilliz

26
cpp/src/cache/GpuCacheMgr.cpp vendored
View File

@ -16,11 +16,13 @@
// under the License.
#include <sstream>
#include "cache/GpuCacheMgr.h"
#include "utils/Log.h"
#include "GpuCacheMgr.h"
#include "server/Config.h"
#include <sstream>
#include <utility>
namespace zilliz {
namespace milvus {
namespace cache {
@ -29,11 +31,11 @@ std::mutex GpuCacheMgr::mutex_;
std::unordered_map<uint64_t, GpuCacheMgrPtr> GpuCacheMgr::instance_;
namespace {
constexpr int64_t G_BYTE = 1024 * 1024 * 1024;
constexpr int64_t G_BYTE = 1024 * 1024 * 1024;
}
GpuCacheMgr::GpuCacheMgr() {
server::Config& config = server::Config::GetInstance();
server::Config &config = server::Config::GetInstance();
Status s;
int32_t gpu_mem_cap;
@ -42,7 +44,7 @@ GpuCacheMgr::GpuCacheMgr() {
SERVER_LOG_ERROR << s.message();
}
int32_t cap = gpu_mem_cap * G_BYTE;
cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL<<32);
cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL << 32);
float gpu_mem_threshold;
s = config.GetCacheConfigGpuMemThreshold(gpu_mem_threshold);
@ -57,7 +59,8 @@ GpuCacheMgr::GpuCacheMgr() {
}
}
GpuCacheMgr* GpuCacheMgr::GetInstance(uint64_t gpu_id) {
GpuCacheMgr *
GpuCacheMgr::GetInstance(uint64_t gpu_id) {
if (instance_.find(gpu_id) == instance_.end()) {
std::lock_guard<std::mutex> lock(mutex_);
if (instance_.find(gpu_id) == instance_.end()) {
@ -70,15 +73,16 @@ GpuCacheMgr* GpuCacheMgr::GetInstance(uint64_t gpu_id) {
}
}
engine::VecIndexPtr GpuCacheMgr::GetIndex(const std::string& key) {
engine::VecIndexPtr
GpuCacheMgr::GetIndex(const std::string &key) {
DataObjPtr obj = GetItem(key);
if(obj != nullptr) {
if (obj != nullptr) {
return obj->data();
}
return nullptr;
}
}
}
}
} // namespace cache
} // namespace milvus
} // namespace zilliz

15
cpp/src/cache/GpuCacheMgr.h vendored
View File

@ -21,6 +21,7 @@
#include <unordered_map>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
@ -30,18 +31,18 @@ class GpuCacheMgr;
using GpuCacheMgrPtr = std::shared_ptr<GpuCacheMgr>;
class GpuCacheMgr : public CacheMgr<DataObjPtr> {
public:
public:
GpuCacheMgr();
static GpuCacheMgr* GetInstance(uint64_t gpu_id);
static GpuCacheMgr *GetInstance(uint64_t gpu_id);
engine::VecIndexPtr GetIndex(const std::string& key);
engine::VecIndexPtr GetIndex(const std::string &key);
private:
private:
static std::mutex mutex_;
static std::unordered_map<uint64_t, GpuCacheMgrPtr> instance_;
};
}
}
}
} // namespace cache
} // namespace milvus
} // namespace zilliz

22
cpp/src/cache/LRU.h vendored
View File

@ -22,6 +22,7 @@
#include <list>
#include <cstddef>
#include <stdexcept>
#include <utility>
namespace zilliz {
namespace milvus {
@ -29,14 +30,15 @@ namespace cache {
template<typename key_t, typename value_t>
class LRU {
public:
public:
typedef typename std::pair<key_t, value_t> key_value_pair_t;
typedef typename std::list<key_value_pair_t>::iterator list_iterator_t;
typedef typename std::list<key_value_pair_t>::reverse_iterator reverse_list_iterator_t;
LRU(size_t max_size) : max_size_(max_size) {}
explicit LRU(size_t max_size) : max_size_(max_size) {
}
void put(const key_t& key, const value_t& value) {
void put(const key_t &key, const value_t &value) {
auto it = cache_items_map_.find(key);
cache_items_list_.push_front(key_value_pair_t(key, value));
if (it != cache_items_map_.end()) {
@ -53,7 +55,7 @@ public:
}
}
const value_t& get(const key_t& key) {
const value_t &get(const key_t &key) {
auto it = cache_items_map_.find(key);
if (it == cache_items_map_.end()) {
throw std::range_error("There is no such key in cache");
@ -63,7 +65,7 @@ public:
}
}
void erase(const key_t& key) {
void erase(const key_t &key) {
auto it = cache_items_map_.find(key);
if (it != cache_items_map_.end()) {
cache_items_list_.erase(it->second);
@ -71,7 +73,7 @@ public:
}
}
bool exists(const key_t& key) const {
bool exists(const key_t &key) const {
return cache_items_map_.find(key) != cache_items_map_.end();
}
@ -101,14 +103,14 @@ public:
cache_items_map_.clear();
}
private:
private:
std::list<key_value_pair_t> cache_items_list_;
std::unordered_map<key_t, list_iterator_t> cache_items_map_;
size_t max_size_;
list_iterator_t iter_;
};
} // cache
} // milvus
} // zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz

11
cpp/src/config/ConfigMgr.cpp
View File

@ -15,18 +15,19 @@
// specific language governing permissions and limitations
// under the License.
#include "ConfigMgr.h"
#include "config/ConfigMgr.h"
#include "YamlConfigMgr.h"
namespace zilliz {
namespace milvus {
namespace server {
ConfigMgr * ConfigMgr::GetInstance() {
ConfigMgr *
ConfigMgr::GetInstance() {
static YamlConfigMgr mgr;
return &mgr;
}
}
}
}
} // namespace server
} // namespace milvus
} // namespace zilliz

14
cpp/src/config/ConfigMgr.h
View File

@ -20,6 +20,8 @@
#include "utils/Error.h"
#include "ConfigNode.h"
#include <string>
namespace zilliz {
namespace milvus {
namespace server {
@ -40,16 +42,16 @@ namespace server {
class ConfigMgr {
public:
static ConfigMgr* GetInstance();
static ConfigMgr *GetInstance();
virtual ErrorCode LoadConfigFile(const std::string &filename) = 0;
virtual void Print() const = 0;//will be deleted
virtual std::string DumpString() const = 0;
virtual const ConfigNode& GetRootNode() const = 0;
virtual ConfigNode& GetRootNode() = 0;
virtual const ConfigNode &GetRootNode() const = 0;
virtual ConfigNode &GetRootNode() = 0;
};
}
}
}
} // namespace server
} // namespace milvus
} // namespace zilliz

80
cpp/src/config/ConfigNode.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "ConfigNode.h"
#include "config/ConfigNode.h"
#include "utils/Error.h"
#include "utils/Log.h"
@ -27,33 +27,34 @@ namespace zilliz {
namespace milvus {
namespace server {
void ConfigNode::Combine(const ConfigNode& target) {
const std::map<std::string, std::string>& kv = target.GetConfig();
for(auto itr = kv.begin(); itr != kv.end(); ++itr){
void
ConfigNode::Combine(const ConfigNode &target) {
const std::map<std::string, std::string> &kv = target.GetConfig();
for (auto itr = kv.begin(); itr != kv.end(); ++itr) {
config_[itr->first] = itr->second;
}
const std::map<std::string, std::vector<std::string> >& sequences = target.GetSequences();
for(auto itr = sequences.begin(); itr != sequences.end(); ++itr){
const std::map<std::string, std::vector<std::string> > &sequences = target.GetSequences();
for (auto itr = sequences.begin(); itr != sequences.end(); ++itr) {
sequences_[itr->first] = itr->second;
}
const std::map<std::string, ConfigNode>& children = target.GetChildren();
for(auto itr = children.begin(); itr != children.end(); ++itr){
const std::map<std::string, ConfigNode> &children = target.GetChildren();
for (auto itr = children.begin(); itr != children.end(); ++itr) {
children_[itr->first] = itr->second;
}
}
//key/value pair config
void
ConfigNode::SetValue(const std::string& key, const std::string& value) {
ConfigNode::SetValue(const std::string &key, const std::string &value) {
config_[key] = value;
}
std::string
ConfigNode::GetValue(const std::string& param_key, const std::string& default_val) const {
ConfigNode::GetValue(const std::string &param_key, const std::string &default_val) const {
auto ref = config_.find(param_key);
if(ref != config_.end()) {
if (ref != config_.end()) {
return ref->second;
}
@ -76,7 +77,7 @@ int32_t
ConfigNode::GetInt32Value(const std::string &param_key, int32_t default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
return (int32_t)std::strtol(val.c_str(), nullptr, 10);
return (int32_t) std::strtol(val.c_str(), nullptr, 10);
} else {
return default_val;
}
@ -112,25 +113,26 @@ ConfigNode::GetDoubleValue(const std::string &param_key, double default_val) con
}
}
const std::map<std::string, std::string>&
const std::map<std::string, std::string> &
ConfigNode::GetConfig() const {
return config_;
};
}
void ConfigNode::ClearConfig() {
void
ConfigNode::ClearConfig() {
config_.clear();
}
//key/object config
void
ConfigNode::AddChild(const std::string& type_name, const ConfigNode& config) {
ConfigNode::AddChild(const std::string &type_name, const ConfigNode &config) {
children_[type_name] = config;
}
ConfigNode
ConfigNode::GetChild(const std::string& type_name) const {
ConfigNode::GetChild(const std::string &type_name) const {
auto ref = children_.find(type_name);
if(ref != children_.end()) {
if (ref != children_.end()) {
return ref->second;
}
@ -138,25 +140,26 @@ ConfigNode::GetChild(const std::string& type_name) const {
return nc;
}
ConfigNode&
ConfigNode &
ConfigNode::GetChild(const std::string &type_name) {
return children_[type_name];
}
void
ConfigNode::GetChildren(ConfigNodeArr& arr) const {
ConfigNode::GetChildren(ConfigNodeArr &arr) const {
arr.clear();
for(auto ref : children_){
for (auto ref : children_) {
arr.push_back(ref.second);
}
}
const std::map<std::string, ConfigNode>&
const std::map<std::string, ConfigNode> &
ConfigNode::GetChildren() const {
return children_;
}
void ConfigNode::ClearChildren() {
void
ConfigNode::ClearChildren() {
children_.clear();
}
@ -169,7 +172,7 @@ ConfigNode::AddSequenceItem(const std::string &key, const std::string &item) {
std::vector<std::string>
ConfigNode::GetSequence(const std::string &key) const {
auto itr = sequences_.find(key);
if(itr != sequences_.end()) {
if (itr != sequences_.end()) {
return itr->second;
} else {
std::vector<std::string> temp;
@ -177,29 +180,30 @@ ConfigNode::GetSequence(const std::string &key) const {
}
}
const std::map<std::string, std::vector<std::string> >&
const std::map<std::string, std::vector<std::string> > &
ConfigNode::GetSequences() const {
return sequences_;
}
void ConfigNode::ClearSequences() {
void
ConfigNode::ClearSequences() {
sequences_.clear();
}
void
ConfigNode::PrintAll(const std::string& prefix) const {
for(auto& elem : config_) {
ConfigNode::PrintAll(const std::string &prefix) const {
for (auto &elem : config_) {
SERVER_LOG_INFO << prefix << elem.first + ": " << elem.second;
}
for(auto& elem : sequences_) {
for (auto &elem : sequences_) {
SERVER_LOG_INFO << prefix << elem.first << ": ";
for(auto& str : elem.second) {
for (auto &str : elem.second) {
SERVER_LOG_INFO << prefix << " - " << str;
}
}
for(auto& elem : children_) {
for (auto &elem : children_) {
SERVER_LOG_INFO << prefix << elem.first << ": ";
elem.second.PrintAll(prefix + " ");
}
@ -209,18 +213,18 @@ std::string
ConfigNode::DumpString(const std::string &prefix) const {
std::stringstream str_buffer;
const std::string endl = "\n";
for(auto& elem : config_) {
for (auto &elem : config_) {
str_buffer << prefix << elem.first << ": " << elem.second << endl;
}
for(auto& elem : sequences_) {
for (auto &elem : sequences_) {
str_buffer << prefix << elem.first << ": " << endl;
for(auto& str : elem.second) {
for (auto &str : elem.second) {
str_buffer << prefix + " - " << str << endl;
}
}
for(auto& elem : children_) {
for (auto &elem : children_) {
str_buffer << prefix << elem.first << ": " << endl;
str_buffer << elem.second.DumpString(prefix + " ") << endl;
}
@ -228,6 +232,6 @@ ConfigNode::DumpString(const std::string &prefix) const {
return str_buffer.str();
}
}
}
}
} // namespace server
} // namespace milvus
} // namespace zilliz

16
cpp/src/config/ConfigNode.h
View File

@ -30,7 +30,7 @@ typedef std::vector<ConfigNode> ConfigNodeArr;
class ConfigNode {
public:
void Combine(const ConfigNode& target);
void Combine(const ConfigNode &target);
//key/value pair config
void SetValue(const std::string &key, const std::string &value);
@ -42,23 +42,23 @@ class ConfigNode {
float GetFloatValue(const std::string &param_key, float default_val = 0.0) const;
double GetDoubleValue(const std::string &param_key, double default_val = 0.0) const;
const std::map<std::string, std::string>& GetConfig() const;
const std::map<std::string, std::string> &GetConfig() const;
void ClearConfig();
//key/object config
void AddChild(const std::string &type_name, const ConfigNode &config);
ConfigNode GetChild(const std::string &type_name) const;
ConfigNode& GetChild(const std::string &type_name);
ConfigNode &GetChild(const std::string &type_name);
void GetChildren(ConfigNodeArr &arr) const;
const std::map<std::string, ConfigNode>& GetChildren() const;
const std::map<std::string, ConfigNode> &GetChildren() const;
void ClearChildren();
//key/sequence config
void AddSequenceItem(const std::string &key, const std::string &item);
std::vector<std::string> GetSequence(const std::string &key) const;
const std::map<std::string, std::vector<std::string> >& GetSequences() const;
const std::map<std::string, std::vector<std::string> > &GetSequences() const;
void ClearSequences();
void PrintAll(const std::string &prefix = "") const;
@ -70,6 +70,6 @@ class ConfigNode {
std::map<std::string, std::vector<std::string> > sequences_;
};
}
}
}
} // namespace server
} // namespace milvus
} // namespace zilliz

57
cpp/src/config/YamlConfigMgr.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "YamlConfigMgr.h"
#include "config/YamlConfigMgr.h"
#include "utils/Log.h"
#include <sys/stat.h>
@ -24,7 +24,8 @@ namespace zilliz {
namespace milvus {
namespace server {
ErrorCode YamlConfigMgr::LoadConfigFile(const std::string &filename) {
ErrorCode
YamlConfigMgr::LoadConfigFile(const std::string &filename) {
struct stat directoryStat;
int statOK = stat(filename.c_str(), &directoryStat);
if (statOK != 0) {
@ -36,36 +37,40 @@ ErrorCode YamlConfigMgr::LoadConfigFile(const std::string &filename) {
node_ = YAML::LoadFile(filename);
LoadConfigNode(node_, config_);
}
catch (YAML::Exception& e) {
SERVER_LOG_ERROR << "Failed to load config file: " << std::string(e.what ());
catch (YAML::Exception &e) {
SERVER_LOG_ERROR << "Failed to load config file: " << std::string(e.what());
return SERVER_UNEXPECTED_ERROR;
}
return SERVER_SUCCESS;
}
void YamlConfigMgr::Print() const {
void
YamlConfigMgr::Print() const {
SERVER_LOG_INFO << "System config content:";
config_.PrintAll();
}
std::string YamlConfigMgr::DumpString() const {
std::string
YamlConfigMgr::DumpString() const {
return config_.DumpString("");
}
const ConfigNode& YamlConfigMgr::GetRootNode() const {
const ConfigNode &
YamlConfigMgr::GetRootNode() const {
return config_;
}
ConfigNode& YamlConfigMgr::GetRootNode() {
ConfigNode &
YamlConfigMgr::GetRootNode() {
return config_;
}
bool
YamlConfigMgr::SetConfigValue(const YAML::Node& node,
const std::string& key,
ConfigNode& config) {
if(node[key].IsDefined ()) {
YamlConfigMgr::SetConfigValue(const YAML::Node &node,
const std::string &key,
ConfigNode &config) {
if (node[key].IsDefined()) {
config.SetValue(key, node[key].as<std::string>());
return true;
}
@ -73,10 +78,10 @@ YamlConfigMgr::SetConfigValue(const YAML::Node& node,
}
bool
YamlConfigMgr::SetChildConfig(const YAML::Node& node,
const std::string& child_name,
ConfigNode& config) {
if(node[child_name].IsDefined ()) {
YamlConfigMgr::SetChildConfig(const YAML::Node &node,
const std::string &child_name,
ConfigNode &config) {
if (node[child_name].IsDefined()) {
ConfigNode sub_config;
LoadConfigNode(node[child_name], sub_config);
config.AddChild(child_name, sub_config);
@ -89,9 +94,9 @@ bool
YamlConfigMgr::SetSequence(const YAML::Node &node,
const std::string &child_name,
ConfigNode &config) {
if(node[child_name].IsDefined ()) {
if (node[child_name].IsDefined()) {
size_t cnt = node[child_name].size();
for(size_t i = 0; i < cnt; i++){
for (size_t i = 0; i < cnt; i++) {
config.AddSequenceItem(child_name, node[child_name][i].as<std::string>());
}
return true;
@ -100,22 +105,22 @@ YamlConfigMgr::SetSequence(const YAML::Node &node,
}
void
YamlConfigMgr::LoadConfigNode(const YAML::Node& node, ConfigNode& config) {
YamlConfigMgr::LoadConfigNode(const YAML::Node &node, ConfigNode &config) {
std::string key;
for (YAML::const_iterator it = node.begin(); it != node.end(); ++it) {
if(!it->first.IsNull()){
if (!it->first.IsNull()) {
key = it->first.as<std::string>();
}
if(node[key].IsScalar()) {
if (node[key].IsScalar()) {
SetConfigValue(node, key, config);
} else if(node[key].IsMap()){
} else if (node[key].IsMap()) {
SetChildConfig(node, key, config);
} else if(node[key].IsSequence()){
} else if (node[key].IsSequence()) {
SetSequence(node, key, config);
}
}
}
}
}
}
} // namespace server
} // namespace milvus
} // namespace zilliz

23
cpp/src/config/YamlConfigMgr.h
View File

@ -21,6 +21,7 @@
#include "ConfigNode.h"
#include "utils/Error.h"
#include <string>
#include <yaml-cpp/yaml.h>
namespace zilliz {
@ -33,15 +34,15 @@ class YamlConfigMgr : public ConfigMgr {
virtual void Print() const;
virtual std::string DumpString() const;
virtual const ConfigNode& GetRootNode() const;
virtual ConfigNode& GetRootNode();
virtual const ConfigNode &GetRootNode() const;
virtual ConfigNode &GetRootNode();
private:
bool SetConfigValue(const YAML::Node& node,
const std::string& key,
ConfigNode& config);
bool SetConfigValue(const YAML::Node &node,
const std::string &key,
ConfigNode &config);
bool SetChildConfig(const YAML::Node& node,
bool SetChildConfig(const YAML::Node &node,
const std::string &name,
ConfigNode &config);
@ -50,15 +51,13 @@ class YamlConfigMgr : public ConfigMgr {
const std::string &child_name,
ConfigNode &config);
void LoadConfigNode(const YAML::Node& node, ConfigNode& config);
void LoadConfigNode(const YAML::Node &node, ConfigNode &config);
private:
YAML::Node node_;
ConfigNode config_;
};
}
}
}
} // namespace server
} // namespace milvus
} // namespace zilliz

4
cpp/src/db/Constants.h
View File

@ -33,8 +33,8 @@ constexpr uint64_t MAX_TABLE_FILE_MEM = 128 * M;
constexpr int VECTOR_TYPE_SIZE = sizeof(float);
static constexpr uint64_t ONE_KB = K;
static constexpr uint64_t ONE_MB = ONE_KB*ONE_KB;
static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB;
static constexpr uint64_t ONE_MB = ONE_KB * ONE_KB;
static constexpr uint64_t ONE_GB = ONE_KB * ONE_MB;
} // namespace engine
} // namespace milvus

48
cpp/src/db/DB.h
View File

@ -25,6 +25,7 @@
#include <string>
#include <memory>
#include <vector>
namespace zilliz {
namespace milvus {
@ -33,46 +34,45 @@ namespace engine {
class Env;
class DB {
public:
public:
DB() = default;
DB(const DB&) = delete;
DB& operator=(const DB&) = delete;
DB(const DB &) = delete;
DB &operator=(const DB &) = delete;
virtual ~DB() = default;
virtual Status Start() = 0;
virtual Status Stop() = 0;
virtual Status CreateTable(meta::TableSchema& table_schema_) = 0;
virtual Status DeleteTable(const std::string& table_id, const meta::DatesT& dates) = 0;
virtual Status DescribeTable(meta::TableSchema& table_schema_) = 0;
virtual Status HasTable(const std::string& table_id, bool& has_or_not_) = 0;
virtual Status AllTables(std::vector<meta::TableSchema>& table_schema_array) = 0;
virtual Status GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0;
virtual Status PreloadTable(const std::string& table_id) = 0;
virtual Status CreateTable(meta::TableSchema &table_schema_) = 0;
virtual Status DeleteTable(const std::string &table_id, const meta::DatesT &dates) = 0;
virtual Status DescribeTable(meta::TableSchema &table_schema_) = 0;
virtual Status HasTable(const std::string &table_id, bool &has_or_not_) = 0;
virtual Status AllTables(std::vector<meta::TableSchema> &table_schema_array) = 0;
virtual Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) = 0;
virtual Status PreloadTable(const std::string &table_id) = 0;
virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0;
virtual Status InsertVectors(const std::string& table_id_,
uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0;
virtual Status InsertVectors(const std::string &table_id_,
uint64_t n, const float *vectors, IDNumbers &vector_ids_) = 0;
virtual Status Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float* vectors, QueryResults& results) = 0;
virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, QueryResults &results) = 0;
virtual Status Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0;
virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, const meta::DatesT &dates, QueryResults &results) = 0;
virtual Status Query(const std::string& table_id, const std::vector<std::string>& file_ids,
uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) = 0;
virtual Status Query(const std::string &table_id, const std::vector<std::string> &file_ids,
uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors,
const meta::DatesT &dates, QueryResults &results) = 0;
virtual Status Size(uint64_t& result) = 0;
virtual Status Size(uint64_t &result) = 0;
virtual Status CreateIndex(const std::string& table_id, const TableIndex& index) = 0;
virtual Status DescribeIndex(const std::string& table_id, TableIndex& index) = 0;
virtual Status DropIndex(const std::string& table_id) = 0;
virtual Status CreateIndex(const std::string &table_id, const TableIndex &index) = 0;
virtual Status DescribeIndex(const std::string &table_id, TableIndex &index) = 0;
virtual Status DropIndex(const std::string &table_id) = 0;
virtual Status DropAll() = 0;
}; // DB
using DBPtr = std::shared_ptr<DB>;

10
cpp/src/db/DBFactory.cpp
View File

@ -16,7 +16,7 @@
// under the License.
#include "DBFactory.h"
#include "db/DBFactory.h"
#include "DBImpl.h"
#include "utils/Exception.h"
#include "meta/MetaFactory.h"
@ -33,14 +33,16 @@ namespace zilliz {
namespace milvus {
namespace engine {
DBOptions DBFactory::BuildOption() {
DBOptions
DBFactory::BuildOption() {
auto meta = MetaFactory::BuildOption();
DBOptions options;
DBOptions options;
options.meta_ = meta;
return options;
}
DBPtr DBFactory::Build(const DBOptions& options) {
DBPtr
DBFactory::Build(const DBOptions &options) {
return std::make_shared<DBImpl>(options);
}

11
cpp/src/db/DBFactory.h
View File

@ -28,13 +28,12 @@ namespace milvus {
namespace engine {
class DBFactory {
public:
public:
static DBOptions BuildOption();
static DBPtr Build(const DBOptions& options);
static DBPtr Build(const DBOptions &options);
};
}
}
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

300
cpp/src/db/DBImpl.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "DBImpl.h"
#include "db/DBImpl.h"
#include "cache/CpuCacheMgr.h"
#include "cache/GpuCacheMgr.h"
#include "engine/EngineFactory.h"
@ -36,6 +36,7 @@
#include <thread>
#include <iostream>
#include <cstring>
#include <algorithm>
#include <boost/filesystem.hpp>
namespace zilliz {
@ -48,10 +49,9 @@ constexpr uint64_t METRIC_ACTION_INTERVAL = 1;
constexpr uint64_t COMPACT_ACTION_INTERVAL = 1;
constexpr uint64_t INDEX_ACTION_INTERVAL = 1;
}
} // namespace
DBImpl::DBImpl(const DBOptions& options)
DBImpl::DBImpl(const DBOptions &options)
: options_(options),
shutting_down_(true),
compact_thread_pool_(1, 1),
@ -68,8 +68,9 @@ DBImpl::~DBImpl() {
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//external api
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Status DBImpl::Start() {
if (!shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::Start() {
if (!shutting_down_.load(std::memory_order_acquire)) {
return Status::OK();
}
@ -77,7 +78,7 @@ Status DBImpl::Start() {
shutting_down_.store(false, std::memory_order_release);
//for distribute version, some nodes are read only
if (options_.mode_ != DBOptions::MODE::READ_ONLY) {
if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) {
ENGINE_LOG_TRACE << "StartTimerTasks";
bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this);
}
@ -85,8 +86,9 @@ Status DBImpl::Start() {
return Status::OK();
}
Status DBImpl::Stop() {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::Stop() {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status::OK();
}
@ -98,7 +100,7 @@ Status DBImpl::Stop() {
//wait compaction/buildindex finish
bg_timer_thread_.join();
if (options_.mode_ != DBOptions::MODE::READ_ONLY) {
if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) {
meta_ptr_->CleanUp();
}
@ -106,12 +108,14 @@ Status DBImpl::Stop() {
return Status::OK();
}
Status DBImpl::DropAll() {
Status
DBImpl::DropAll() {
return meta_ptr_->DropAll();
}
Status DBImpl::CreateTable(meta::TableSchema& table_schema) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::CreateTable(meta::TableSchema &table_schema) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -120,8 +124,9 @@ Status DBImpl::CreateTable(meta::TableSchema& table_schema) {
return meta_ptr_->CreateTable(temp_schema);
}
Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::DeleteTable(const std::string &table_id, const meta::DatesT &dates) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -144,8 +149,9 @@ Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& date
return Status::OK();
}
Status DBImpl::DescribeTable(meta::TableSchema& table_schema) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::DescribeTable(meta::TableSchema &table_schema) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -154,24 +160,27 @@ Status DBImpl::DescribeTable(meta::TableSchema& table_schema) {
return stat;
}
Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::HasTable(const std::string &table_id, bool &has_or_not) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
return meta_ptr_->HasTable(table_id, has_or_not);
}
Status DBImpl::AllTables(std::vector<meta::TableSchema>& table_schema_array) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::AllTables(std::vector<meta::TableSchema> &table_schema_array) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
return meta_ptr_->AllTables(table_schema_array);
}
Status DBImpl::PreloadTable(const std::string &table_id) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::PreloadTable(const std::string &table_id) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -189,10 +198,14 @@ Status DBImpl::PreloadTable(const std::string &table_id) {
int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
int64_t available_size = cache_total - cache_usage;
for(auto &day_files : files) {
for (auto &day_files : files) {
for (auto &file : day_files.second) {
ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, (MetricType)file.metric_type_, file.nlist_);
if(engine == nullptr) {
ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_,
file.location_,
(EngineType) file.engine_type_,
(MetricType) file.metric_type_,
file.nlist_);
if (engine == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type";
return Status(DB_ERROR, "Invalid engine type");
}
@ -215,33 +228,37 @@ Status DBImpl::PreloadTable(const std::string &table_id) {
return Status::OK();
}
Status DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
return meta_ptr_->UpdateTableFlag(table_id, flag);
}
Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::GetTableRowCount(const std::string &table_id, uint64_t &row_count) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
return meta_ptr_->Count(table_id, row_count);
}
Status DBImpl::InsertVectors(const std::string& table_id_,
uint64_t n, const float* vectors, IDNumbers& vector_ids_) {
Status
DBImpl::InsertVectors(const std::string &table_id_,
uint64_t n, const float *vectors, IDNumbers &vector_ids_) {
// ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache";
if (shutting_down_.load(std::memory_order_acquire)){
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
Status status;
zilliz::milvus::server::CollectInsertMetrics metrics(n, status);
status = mem_mgr_->InsertVectors(table_id_, n, vectors, vector_ids_);
// std::chrono::microseconds time_span = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time);
// std::chrono::microseconds time_span =
// std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time);
// double average_time = double(time_span.count()) / n;
// ENGINE_LOG_DEBUG << "Insert vectors to cache finished";
@ -249,14 +266,15 @@ Status DBImpl::InsertVectors(const std::string& table_id_,
return status;
}
Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) {
Status
DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) {
{
std::unique_lock<std::mutex> lock(build_index_mutex_);
//step 1: check index difference
TableIndex old_index;
auto status = DescribeIndex(table_id, old_index);
if(!status.ok()) {
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to get table index info for table: " << table_id;
return status;
}
@ -264,7 +282,7 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
//step 2: update index info
TableIndex new_index = index;
new_index.metric_type_ = old_index.metric_type_;//dont change metric type, it was defined by CreateTable
if(!utils::IsSameIndex(old_index, new_index)) {
if (!utils::IsSameIndex(old_index, new_index)) {
DropIndex(table_id);
status = meta_ptr_->UpdateTableIndex(table_id, new_index);
@ -283,18 +301,18 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
//for IDMAP type, only wait all NEW file converted to RAW file
//for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files
std::vector<int> file_types;
if(index.engine_type_ == (int)EngineType::FAISS_IDMAP) {
if (index.engine_type_ == (int) EngineType::FAISS_IDMAP) {
file_types = {
(int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE,
(int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE,
};
} else {
file_types = {
(int) meta::TableFileSchema::RAW,
(int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE,
(int) meta::TableFileSchema::NEW_INDEX,
(int) meta::TableFileSchema::TO_INDEX,
(int) meta::TableFileSchema::RAW,
(int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE,
(int) meta::TableFileSchema::NEW_INDEX,
(int) meta::TableFileSchema::TO_INDEX,
};
}
@ -304,11 +322,11 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
while (!file_ids.empty()) {
ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times;
if(index.engine_type_ != (int)EngineType::FAISS_IDMAP) {
if (index.engine_type_ != (int) EngineType::FAISS_IDMAP) {
status = meta_ptr_->UpdateTableFilesToIndex(table_id);
}
std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100)));
std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10 * 1000, times * 100)));
status = meta_ptr_->FilesByType(table_id, file_types, file_ids);
times++;
}
@ -316,18 +334,21 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
return Status::OK();
}
Status DBImpl::DescribeIndex(const std::string& table_id, TableIndex& index) {
Status
DBImpl::DescribeIndex(const std::string &table_id, TableIndex &index) {
return meta_ptr_->DescribeTableIndex(table_id, index);
}
Status DBImpl::DropIndex(const std::string& table_id) {
Status
DBImpl::DropIndex(const std::string &table_id) {
ENGINE_LOG_DEBUG << "Drop index for table: " << table_id;
return meta_ptr_->DropTableIndex(table_id);
}
Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, QueryResults &results) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, QueryResults &results) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -337,9 +358,10 @@ Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint6
return result;
}
Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float* vectors, const meta::DatesT& dates, QueryResults& results) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, const meta::DatesT &dates, QueryResults &results) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -364,10 +386,11 @@ Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint6
return status;
}
Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>& file_ids,
uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::Query(const std::string &table_id, const std::vector<std::string> &file_ids,
uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors,
const meta::DatesT &dates, QueryResults &results) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -395,7 +418,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>
}
}
if(file_id_array.empty()) {
if (file_id_array.empty()) {
return Status(DB_ERROR, "Invalid file id");
}
@ -405,36 +428,37 @@ Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>
return status;
}
Status DBImpl::Size(uint64_t& result) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::Size(uint64_t &result) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
return meta_ptr_->Size(result);
return meta_ptr_->Size(result);
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//internal methods
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files,
uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) {
using namespace scheduler;
Status
DBImpl::QueryAsync(const std::string &table_id, const meta::TableFilesSchema &files,
uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors,
const meta::DatesT &dates, QueryResults &results) {
server::CollectQueryMetrics metrics(nq);
TimeRecorder rc("");
//step 1: get files to search
ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: " << dates.size();
SearchJobPtr job = std::make_shared<SearchJob>(0, k, nq, nprobe, vectors);
for (auto &file : files) {
TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file);
ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: "
<< dates.size();
scheduler::SearchJobPtr job = std::make_shared<scheduler::SearchJob>(0, k, nq, nprobe, vectors);
for (auto &file : files) {
scheduler::TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file);
job->AddIndexFile(file_ptr);
}
//step 2: put search task to scheduler
JobMgrInst::GetInstance()->Put(job);
scheduler::JobMgrInst::GetInstance()->Put(job);
job->WaitResult();
if (!job->GetStatus().ok()) {
return job->GetStatus();
@ -453,9 +477,12 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch
// double search_percent = search_cost/total_cost;
// double reduce_percent = reduce_cost/total_cost;
//
// ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info << " percent: " << load_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info << " percent: " << search_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info << " percent: " << reduce_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info
// << " percent: " << load_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info
// << " percent: " << search_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info
// << " percent: " << reduce_percent*100 << "%";
// } else {
// ENGINE_LOG_DEBUG << "Engine load cost: " << load_info
// << " search cost: " << search_info
@ -469,11 +496,12 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch
return Status::OK();
}
void DBImpl::BackgroundTimerTask() {
void
DBImpl::BackgroundTimerTask() {
Status status;
server::SystemInfo::GetInstance().Init();
while (true) {
if (shutting_down_.load(std::memory_order_acquire)){
if (shutting_down_.load(std::memory_order_acquire)) {
WaitMergeFileFinish();
WaitBuildIndexFinish();
@ -489,24 +517,27 @@ void DBImpl::BackgroundTimerTask() {
}
}
void DBImpl::WaitMergeFileFinish() {
void
DBImpl::WaitMergeFileFinish() {
std::lock_guard<std::mutex> lck(compact_result_mutex_);
for(auto& iter : compact_thread_results_) {
for (auto &iter : compact_thread_results_) {
iter.wait();
}
}
void DBImpl::WaitBuildIndexFinish() {
void
DBImpl::WaitBuildIndexFinish() {
std::lock_guard<std::mutex> lck(index_result_mutex_);
for(auto& iter : index_thread_results_) {
for (auto &iter : index_thread_results_) {
iter.wait();
}
}
void DBImpl::StartMetricTask() {
void
DBImpl::StartMetricTask() {
static uint64_t metric_clock_tick = 0;
metric_clock_tick++;
if(metric_clock_tick%METRIC_ACTION_INTERVAL != 0) {
if (metric_clock_tick % METRIC_ACTION_INTERVAL != 0) {
return;
}
@ -515,7 +546,7 @@ void DBImpl::StartMetricTask() {
server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL);
int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
int64_t cache_total = cache::CpuCacheMgr::GetInstance()->CacheCapacity();
server::Metrics::GetInstance().CpuCacheUsageGaugeSet(cache_usage*100/cache_total);
server::Metrics::GetInstance().CpuCacheUsageGaugeSet(cache_usage * 100 / cache_total);
server::Metrics::GetInstance().GpuCacheUsageGaugeSet();
uint64_t size;
Size(size);
@ -533,25 +564,27 @@ void DBImpl::StartMetricTask() {
ENGINE_LOG_TRACE << "Metric task finished";
}
Status DBImpl::MemSerialize() {
Status
DBImpl::MemSerialize() {
std::lock_guard<std::mutex> lck(mem_serialize_mutex_);
std::set<std::string> temp_table_ids;
mem_mgr_->Serialize(temp_table_ids);
for(auto& id : temp_table_ids) {
for (auto &id : temp_table_ids) {
compact_table_ids_.insert(id);
}
if(!temp_table_ids.empty()) {
if (!temp_table_ids.empty()) {
SERVER_LOG_DEBUG << "Insert cache serialized";
}
return Status::OK();
}
void DBImpl::StartCompactionTask() {
void
DBImpl::StartCompactionTask() {
static uint64_t compact_clock_tick = 0;
compact_clock_tick++;
if(compact_clock_tick%COMPACT_ACTION_INTERVAL != 0) {
if (compact_clock_tick % COMPACT_ACTION_INTERVAL != 0) {
return;
}
@ -580,8 +613,9 @@ void DBImpl::StartCompactionTask() {
}
}
Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
const meta::TableFilesSchema& files) {
Status
DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date,
const meta::TableFilesSchema &files) {
ENGINE_LOG_DEBUG << "Merge files for table: " << table_id;
//step 1: create table file
@ -598,13 +632,13 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
//step 2: merge files
ExecutionEnginePtr index =
EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_,
(MetricType)table_file.metric_type_, table_file.nlist_);
EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType) table_file.engine_type_,
(MetricType) table_file.metric_type_, table_file.nlist_);
meta::TableFilesSchema updated;
long index_size = 0;
int64_t index_size = 0;
for (auto& file : files) {
for (auto &file : files) {
server::CollectMergeFilesMetrics metrics;
index->Merge(file.location_);
@ -620,7 +654,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
//step 3: serialize to disk
try {
index->Serialize();
} catch (std::exception& ex) {
} catch (std::exception &ex) {
//typical error: out of disk space or permition denied
std::string msg = "Serialize merged index encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
@ -638,7 +672,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
//step 4: update table files state
//if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
//else set file type to RAW, no need to build index
if (table_file.engine_type_ != (int)EngineType::FAISS_IDMAP) {
if (table_file.engine_type_ != (int) EngineType::FAISS_IDMAP) {
table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
} else {
@ -649,16 +683,17 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
updated.push_back(table_file);
status = meta_ptr_->UpdateTableFiles(updated);
ENGINE_LOG_DEBUG << "New merged file " << table_file.file_id_ <<
" of size " << index->PhysicalSize() << " bytes";
" of size " << index->PhysicalSize() << " bytes";
if(options_.insert_cache_immediately_) {
if (options_.insert_cache_immediately_) {
index->Cache();
}
return status;
}
Status DBImpl::BackgroundMergeFiles(const std::string& table_id) {
Status
DBImpl::BackgroundMergeFiles(const std::string &table_id) {
meta::DatePartionedTableFilesSchema raw_files;
auto status = meta_ptr_->FilesToMerge(table_id, raw_files);
if (!status.ok()) {
@ -667,7 +702,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) {
}
bool has_merge = false;
for (auto& kv : raw_files) {
for (auto &kv : raw_files) {
auto files = kv.second;
if (files.size() < options_.merge_trigger_number_) {
ENGINE_LOG_DEBUG << "Files number not greater equal than merge trigger number, skip merge action";
@ -676,7 +711,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) {
has_merge = true;
MergeFiles(table_id, kv.first, kv.second);
if (shutting_down_.load(std::memory_order_acquire)){
if (shutting_down_.load(std::memory_order_acquire)) {
ENGINE_LOG_DEBUG << "Server will shutdown, skip merge action for table: " << table_id;
break;
}
@ -685,17 +720,18 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) {
return Status::OK();
}
void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
void
DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
ENGINE_LOG_TRACE << " Background compaction thread start";
Status status;
for (auto& table_id : table_ids) {
for (auto &table_id : table_ids) {
status = BackgroundMergeFiles(table_id);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Merge files for table " << table_id << " failed: " << status.ToString();
}
if (shutting_down_.load(std::memory_order_acquire)){
if (shutting_down_.load(std::memory_order_acquire)) {
ENGINE_LOG_DEBUG << "Server will shutdown, skip merge action";
break;
}
@ -703,8 +739,8 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
meta_ptr_->Archive();
int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes
if (options_.mode_ == DBOptions::MODE::CLUSTER) {
int ttl = 5 * meta::M_SEC;//default: file will be deleted after 5 minutes
if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) {
ttl = meta::D_SEC;
}
meta_ptr_->CleanUpFilesWithTTL(ttl);
@ -712,10 +748,11 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
ENGINE_LOG_TRACE << " Background compaction thread exit";
}
void DBImpl::StartBuildIndexTask(bool force) {
void
DBImpl::StartBuildIndexTask(bool force) {
static uint64_t index_clock_tick = 0;
index_clock_tick++;
if(!force && (index_clock_tick%INDEX_ACTION_INTERVAL != 0)) {
if (!force && (index_clock_tick % INDEX_ACTION_INTERVAL != 0)) {
return;
}
@ -740,11 +777,12 @@ void DBImpl::StartBuildIndexTask(bool force) {
}
}
Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
Status
DBImpl::BuildIndex(const meta::TableFileSchema &file) {
ExecutionEnginePtr to_index =
EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_,
(MetricType)file.metric_type_, file.nlist_);
if(to_index == nullptr) {
EngineFactory::Build(file.dimension_, file.location_, (EngineType) file.engine_type_,
(MetricType) file.metric_type_, file.nlist_);
if (to_index == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type";
return Status(DB_ERROR, "Invalid engine type");
}
@ -761,7 +799,8 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
meta::TableFileSchema table_file;
table_file.table_id_ = file.table_id_;
table_file.date_ = file.date_;
table_file.file_type_ = meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path
table_file.file_type_ =
meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path
status = meta_ptr_->CreateTableFile(table_file);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to create table file: " << status.ToString();
@ -773,16 +812,16 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
try {
server::CollectBuildIndexMetrics metrics;
index = to_index->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_);
index = to_index->BuildIndex(table_file.location_, (EngineType) table_file.engine_type_);
if (index == nullptr) {
table_file.file_type_ = meta::TableFileSchema::TO_DELETE;
status = meta_ptr_->UpdateTableFile(table_file);
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_
<< " to to_delete";
return status;
}
} catch (std::exception& ex) {
} catch (std::exception &ex) {
//typical error: out of gpu memory
std::string msg = "BuildIndex encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
@ -791,7 +830,8 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
status = meta_ptr_->UpdateTableFile(table_file);
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" << std::endl;
std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough"
<< std::endl;
return Status(DB_ERROR, msg);
}
@ -799,7 +839,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
//step 4: if table has been deleted, dont save index file
bool has_table = false;
meta_ptr_->HasTable(file.table_id_, has_table);
if(!has_table) {
if (!has_table) {
meta_ptr_->DeleteTableFiles(file.table_id_);
return Status::OK();
}
@ -807,7 +847,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
//step 5: save index file
try {
index->Serialize();
} catch (std::exception& ex) {
} catch (std::exception &ex) {
//typical error: out of disk space or permition denied
std::string msg = "Serialize index encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
@ -817,7 +857,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
std::cout << "ERROR: failed to persist index file: " << table_file.location_
<< ", possible out of disk space" << std::endl;
<< ", possible out of disk space" << std::endl;
return Status(DB_ERROR, msg);
}
@ -832,12 +872,12 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
meta::TableFilesSchema update_files = {table_file, origin_file};
status = meta_ptr_->UpdateTableFiles(update_files);
if(status.ok()) {
if (status.ok()) {
ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size "
<< index->PhysicalSize() << " bytes"
<< " from file " << origin_file.file_id_;
if(options_.insert_cache_immediately_) {
if (options_.insert_cache_immediately_) {
index->Cache();
}
} else {
@ -850,8 +890,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
status = meta_ptr_->UpdateTableFile(table_file);
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
}
} catch (std::exception& ex) {
} catch (std::exception &ex) {
std::string msg = "Build index encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
return Status(DB_ERROR, msg);
@ -860,20 +899,21 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
return Status::OK();
}
void DBImpl::BackgroundBuildIndex() {
void
DBImpl::BackgroundBuildIndex() {
ENGINE_LOG_TRACE << "Background build index thread start";
std::unique_lock<std::mutex> lock(build_index_mutex_);
meta::TableFilesSchema to_index_files;
meta_ptr_->FilesToIndex(to_index_files);
Status status;
for (auto& file : to_index_files) {
for (auto &file : to_index_files) {
status = BuildIndex(file);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString();
}
if (shutting_down_.load(std::memory_order_acquire)){
if (shutting_down_.load(std::memory_order_acquire)) {
ENGINE_LOG_DEBUG << "Server will shutdown, skip build index action";
break;
}

12
cpp/src/db/DBImpl.h
View File

@ -29,7 +29,8 @@
#include <thread>
#include <list>
#include <set>
#include <vector>
#include <string>
namespace zilliz {
namespace milvus {
@ -68,11 +69,11 @@ class DBImpl : public DB {
Status InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override;
Status CreateIndex(const std::string& table_id, const TableIndex& index) override;
Status CreateIndex(const std::string &table_id, const TableIndex &index) override;
Status DescribeIndex(const std::string& table_id, TableIndex& index) override;
Status DescribeIndex(const std::string &table_id, TableIndex &index) override;
Status DropIndex(const std::string& table_id) override;
Status DropIndex(const std::string &table_id) override;
Status Query(const std::string &table_id,
uint64_t k,
@ -123,7 +124,7 @@ class DBImpl : public DB {
Status BackgroundMergeFiles(const std::string &table_id);
void BackgroundCompaction(std::set<std::string> table_ids);
void StartBuildIndexTask(bool force=false);
void StartBuildIndexTask(bool force = false);
void BackgroundBuildIndex();
Status BuildIndex(const meta::TableFileSchema &);
@ -151,7 +152,6 @@ class DBImpl : public DB {
std::list<std::future<void>> index_thread_results_;
std::mutex build_index_mutex_;
}; // DBImpl

22
cpp/src/db/IDGenerator.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "IDGenerator.h"
#include "db/IDGenerator.h"
#include <chrono>
#include <assert.h>
@ -29,16 +29,18 @@ IDGenerator::~IDGenerator() = default;
constexpr size_t SimpleIDGenerator::MAX_IDS_PER_MICRO;
IDNumber SimpleIDGenerator::GetNextIDNumber() {
IDNumber
SimpleIDGenerator::GetNextIDNumber() {
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
now.time_since_epoch()).count();
return micros * MAX_IDS_PER_MICRO;
}
void SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) {
void
SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) {
if (n > MAX_IDS_PER_MICRO) {
NextIDNumbers(n-MAX_IDS_PER_MICRO, ids);
NextIDNumbers(n - MAX_IDS_PER_MICRO, ids);
NextIDNumbers(MAX_IDS_PER_MICRO, ids);
return;
}
@ -48,20 +50,20 @@ void SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) {
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
now.time_since_epoch()).count();
micros *= MAX_IDS_PER_MICRO;
for (int pos=0; pos<n; ++pos) {
ids.push_back(micros+pos);
for (int pos = 0; pos < n; ++pos) {
ids.push_back(micros + pos);
}
}
void SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers& ids) {
void
SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers &ids) {
ids.clear();
NextIDNumbers(n, ids);
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

2
cpp/src/db/IDGenerator.h
View File

@ -22,7 +22,6 @@
#include <cstddef>
#include <vector>
namespace zilliz {
namespace milvus {
namespace engine {
@ -55,7 +54,6 @@ class SimpleIDGenerator : public IDGenerator {
NextIDNumbers(size_t n, IDNumbers &ids);
static constexpr size_t MAX_IDS_PER_MICRO = 1000;
}; // SimpleIDGenerator

23
cpp/src/db/Options.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "Options.h"
#include "db/Options.h"
#include "utils/Exception.h"
#include "utils/Log.h"
@ -27,18 +27,20 @@ namespace zilliz {
namespace milvus {
namespace engine {
ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) {
ArchiveConf::ArchiveConf(const std::string &type, const std::string &criterias) {
ParseType(type);
ParseCritirias(criterias);
}
void ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT& criterial) {
for(auto& pair : criterial) {
void
ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT &criterial) {
for (auto &pair : criterial) {
criterias_[pair.first] = pair.second;
}
}
void ArchiveConf::ParseCritirias(const std::string& criterias) {
void
ArchiveConf::ParseCritirias(const std::string &criterias) {
std::stringstream ss(criterias);
std::vector<std::string> tokens;
@ -48,8 +50,8 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
return;
}
for (auto& token : tokens) {
if(token.empty()) {
for (auto &token : tokens) {
if (token.empty()) {
continue;
}
@ -67,12 +69,12 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
auto value = std::stoi(kv[1]);
criterias_[kv[0]] = value;
}
catch (std::out_of_range&){
catch (std::out_of_range &) {
std::string msg = "Out of range: '" + kv[1] + "'";
ENGINE_LOG_ERROR << msg;
throw InvalidArgumentException(msg);
}
catch (...){
catch (...) {
std::string msg = "Invalid argument: '" + kv[1] + "'";
ENGINE_LOG_ERROR << msg;
throw InvalidArgumentException(msg);
@ -80,7 +82,8 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
}
}
void ArchiveConf::ParseType(const std::string& type) {
void
ArchiveConf::ParseType(const std::string &type) {
if (type != "delete" && type != "swap") {
std::string msg = "Invalid argument: type='" + type + "'";
throw InvalidArgumentException(msg);

31
cpp/src/db/Options.h
View File

@ -30,22 +30,27 @@ namespace engine {
class Env;
static const char* ARCHIVE_CONF_DISK = "disk";
static const char* ARCHIVE_CONF_DAYS = "days";
static const char *ARCHIVE_CONF_DISK = "disk";
static const char *ARCHIVE_CONF_DAYS = "days";
struct ArchiveConf {
using CriteriaT = std::map<std::string, int>;
ArchiveConf(const std::string& type, const std::string& criterias = std::string());
explicit ArchiveConf(const std::string &type, const std::string &criterias = std::string());
const std::string& GetType() const { return type_; }
const CriteriaT GetCriterias() const { return criterias_; }
const std::string &GetType() const {
return type_;
}
void SetCriterias(const ArchiveConf::CriteriaT& criterial);
const CriteriaT GetCriterias() const {
return criterias_;
}
private:
void ParseCritirias(const std::string& type);
void ParseType(const std::string& criterias);
void SetCriterias(const ArchiveConf::CriteriaT &criterial);
private:
void ParseCritirias(const std::string &type);
void ParseType(const std::string &criterias);
std::string type_;
CriteriaT criterias_;
@ -60,12 +65,12 @@ struct DBMetaOptions {
struct DBOptions {
typedef enum {
SINGLE,
CLUSTER,
READ_ONLY
SINGLE = 0,
CLUSTER_READONLY,
CLUSTER_WRITABLE
} MODE;
uint16_t merge_trigger_number_ = 2;
uint16_t merge_trigger_number_ = 2;
DBMetaOptions meta_;
int mode_ = MODE::SINGLE;

9
cpp/src/db/Types.h
View File

@ -21,22 +21,23 @@
#include <vector>
#include <stdint.h>
#include <utility>
namespace zilliz {
namespace milvus {
namespace engine {
typedef long IDNumber;
typedef IDNumber* IDNumberPtr;
typedef int64_t IDNumber;
typedef IDNumber *IDNumberPtr;
typedef std::vector<IDNumber> IDNumbers;
typedef std::vector<std::pair<IDNumber, double>> QueryResult;
typedef std::vector<QueryResult> QueryResults;
struct TableIndex {
int32_t engine_type_ = (int)EngineType::FAISS_IDMAP;
int32_t engine_type_ = (int) EngineType::FAISS_IDMAP;
int32_t nlist_ = 16384;
int32_t metric_type_ = (int)MetricType::L2;
int32_t metric_type_ = (int) MetricType::L2;
};
} // namespace engine

74
cpp/src/db/Utils.cpp
View File

@ -15,13 +15,14 @@
// specific language governing permissions and limitations
// under the License.
#include "Utils.h"
#include "db/Utils.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include <mutex>
#include <chrono>
#include <regex>
#include <vector>
#include <boost/filesystem.hpp>
namespace zilliz {
@ -31,23 +32,25 @@ namespace utils {
namespace {
const char* TABLES_FOLDER = "/tables/";
const char *TABLES_FOLDER = "/tables/";
uint64_t index_file_counter = 0;
std::mutex index_file_counter_mutex;
std::string ConstructParentFolder(const std::string& db_path, const meta::TableFileSchema& table_file) {
std::string
ConstructParentFolder(const std::string &db_path, const meta::TableFileSchema &table_file) {
std::string table_path = db_path + TABLES_FOLDER + table_file.table_id_;
std::string partition_path = table_path + "/" + std::to_string(table_file.date_);
return partition_path;
}
std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::TableFileSchema& table_file) {
std::string
GetTableFileParentFolder(const DBMetaOptions &options, const meta::TableFileSchema &table_file) {
uint64_t path_count = options.slave_paths_.size() + 1;
std::string target_path = options.path_;
uint64_t index = 0;
if(meta::TableFileSchema::NEW_INDEX == table_file.file_type_) {
if (meta::TableFileSchema::NEW_INDEX == table_file.file_type_) {
// index file is large file and to be persisted permanently
// we need to distribute index files to each db_path averagely
// round robin according to a file counter
@ -67,17 +70,19 @@ std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::T
return ConstructParentFolder(target_path, table_file);
}
}
} // namespace
long GetMicroSecTimeStamp() {
int64_t
GetMicroSecTimeStamp() {
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
now.time_since_epoch()).count();
return micros;
}
Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id) {
Status
CreateTablePath(const DBMetaOptions &options, const std::string &table_id) {
std::string db_path = options.path_;
std::string table_path = db_path + TABLES_FOLDER + table_id;
auto status = server::CommonUtil::CreateDirectory(table_path);
@ -86,7 +91,7 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
return status;
}
for(auto& path : options.slave_paths_) {
for (auto &path : options.slave_paths_) {
table_path = path + TABLES_FOLDER + table_id;
status = server::CommonUtil::CreateDirectory(table_path);
if (!status.ok()) {
@ -98,17 +103,18 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
return Status::OK();
}
Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force) {
Status
DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force) {
std::vector<std::string> paths = options.slave_paths_;
paths.push_back(options.path_);
for(auto& path : paths) {
for (auto &path : paths) {
std::string table_path = path + TABLES_FOLDER + table_id;
if(force) {
if (force) {
boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
} else if(boost::filesystem::exists(table_path) &&
boost::filesystem::is_empty(table_path)) {
} else if (boost::filesystem::exists(table_path) &&
boost::filesystem::is_empty(table_path)) {
boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
}
@ -117,7 +123,8 @@ Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id
return Status::OK();
}
Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
Status
CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) {
std::string parent_path = GetTableFileParentFolder(options, table_file);
auto status = server::CommonUtil::CreateDirectory(parent_path);
@ -131,17 +138,18 @@ Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema&
return Status::OK();
}
Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
Status
GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) {
std::string parent_path = ConstructParentFolder(options.path_, table_file);
std::string file_path = parent_path + "/" + table_file.file_id_;
if(boost::filesystem::exists(file_path)) {
if (boost::filesystem::exists(file_path)) {
table_file.location_ = file_path;
return Status::OK();
} else {
for(auto& path : options.slave_paths_) {
for (auto &path : options.slave_paths_) {
parent_path = ConstructParentFolder(path, table_file);
file_path = parent_path + "/" + table_file.file_id_;
if(boost::filesystem::exists(file_path)) {
if (boost::filesystem::exists(file_path)) {
table_file.location_ = file_path;
return Status::OK();
}
@ -155,49 +163,55 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab
return Status(DB_ERROR, msg);
}
Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
Status
DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) {
utils::GetTableFilePath(options, table_file);
boost::filesystem::remove(table_file.location_);
return Status::OK();
}
bool IsSameIndex(const TableIndex& index1, const TableIndex& index2) {
bool
IsSameIndex(const TableIndex &index1, const TableIndex &index2) {
return index1.engine_type_ == index2.engine_type_
&& index1.nlist_ == index2.nlist_
&& index1.metric_type_ == index2.metric_type_;
}
meta::DateT GetDate(const std::time_t& t, int day_delta) {
meta::DateT
GetDate(const std::time_t &t, int day_delta) {
struct tm ltm;
localtime_r(&t, &ltm);
if (day_delta > 0) {
do {
++ltm.tm_mday;
--day_delta;
} while(day_delta > 0);
} while (day_delta > 0);
mktime(&ltm);
} else if (day_delta < 0) {
do {
--ltm.tm_mday;
++day_delta;
} while(day_delta < 0);
} while (day_delta < 0);
mktime(&ltm);
} else {
ltm.tm_mday;
}
return ltm.tm_year*10000 + ltm.tm_mon*100 + ltm.tm_mday;
return ltm.tm_year * 10000 + ltm.tm_mon * 100 + ltm.tm_mday;
}
meta::DateT GetDateWithDelta(int day_delta) {
meta::DateT
GetDateWithDelta(int day_delta) {
return GetDate(std::time(nullptr), day_delta);
}
meta::DateT GetDate() {
meta::DateT
GetDate() {
return GetDate(std::time(nullptr), 0);
}
// URI format: dialect://username:password@host:port/database
Status ParseMetaUri(const std::string& uri, MetaUriInfo& info) {
Status
ParseMetaUri(const std::string &uri, MetaUriInfo &info) {
std::string dialect_regex = "(.*)";
std::string username_tegex = "(.*)";
std::string password_regex = "(.*)";
@ -205,7 +219,7 @@ Status ParseMetaUri(const std::string& uri, MetaUriInfo& info) {
std::string port_regex = "(.*)";
std::string db_name_regex = "(.*)";
std::string uri_regex_str =
dialect_regex + "\\:\\/\\/" +
dialect_regex + "\\:\\/\\/" +
username_tegex + "\\:" +
password_regex + "\\@" +
host_regex + "\\:" +

33
cpp/src/db/Utils.h
View File

@ -29,20 +29,30 @@ namespace milvus {
namespace engine {
namespace utils {
long GetMicroSecTimeStamp();
int64_t
GetMicroSecTimeStamp();
Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id);
Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force = true);
Status
CreateTablePath(const DBMetaOptions &options, const std::string &table_id);
Status
DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force = true);
Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
Status
CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file);
Status
GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file);
Status
DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file);
bool IsSameIndex(const TableIndex& index1, const TableIndex& index2);
bool
IsSameIndex(const TableIndex &index1, const TableIndex &index2);
meta::DateT GetDate(const std::time_t &t, int day_delta = 0);
meta::DateT GetDate();
meta::DateT GetDateWithDelta(int day_delta);
meta::DateT
GetDate(const std::time_t &t, int day_delta = 0);
meta::DateT
GetDate();
meta::DateT
GetDateWithDelta(int day_delta);
struct MetaUriInfo {
std::string dialect_;
@ -53,7 +63,8 @@ struct MetaUriInfo {
std::string db_name_;
};
Status ParseMetaUri(const std::string& uri, MetaUriInfo& info);
Status
ParseMetaUri(const std::string &uri, MetaUriInfo &info);
} // namespace utils
} // namespace engine

19
cpp/src/db/engine/EngineFactory.cpp
View File

@ -15,10 +15,12 @@
// specific language governing permissions and limitations
// under the License.
#include "EngineFactory.h"
#include "ExecutionEngineImpl.h"
#include "db/engine/EngineFactory.h"
#include "db/engine/ExecutionEngineImpl.h"
#include "utils/Log.h"
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
@ -29,20 +31,19 @@ EngineFactory::Build(uint16_t dimension,
EngineType index_type,
MetricType metric_type,
int32_t nlist) {
if(index_type == EngineType::INVALID) {
if (index_type == EngineType::INVALID) {
ENGINE_LOG_ERROR << "Unsupported engine type";
return nullptr;
}
ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int)index_type;
ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int) index_type;
ExecutionEnginePtr execution_engine_ptr =
std::make_shared<ExecutionEngineImpl>(dimension, location, index_type, metric_type, nlist);
std::make_shared<ExecutionEngineImpl>(dimension, location, index_type, metric_type, nlist);
execution_engine_ptr->Init();
return execution_engine_ptr;
}
}
}
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

13
cpp/src/db/engine/EngineFactory.h
View File

@ -21,19 +21,22 @@
#include "ExecutionEngine.h"
#include "utils/Status.h"
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
class EngineFactory {
public:
public:
static ExecutionEnginePtr Build(uint16_t dimension,
const std::string& location,
const std::string &location,
EngineType index_type,
MetricType metric_type,
int32_t nlist);
};
}
}
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

23
cpp/src/db/engine/ExecutionEngine.h
View File

@ -21,6 +21,7 @@
#include <vector>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
@ -41,9 +42,8 @@ enum class MetricType {
};
class ExecutionEngine {
public:
virtual Status AddWithIds(long n, const float *xdata, const long *xids) = 0;
public:
virtual Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) = 0;
virtual size_t Count() const = 0;
@ -63,16 +63,16 @@ public:
virtual std::shared_ptr<ExecutionEngine> Clone() = 0;
virtual Status Merge(const std::string& location) = 0;
virtual Status Merge(const std::string &location) = 0;
virtual Status Search(long n,
const float *data,
long k,
long nprobe,
float *distances,
long *labels) const = 0;
virtual Status Search(int64_t n,
const float *data,
int64_t k,
int64_t nprobe,
float *distances,
int64_t *labels) const = 0;
virtual std::shared_ptr<ExecutionEngine> BuildIndex(const std::string& location, EngineType engine_type) = 0;
virtual std::shared_ptr<ExecutionEngine> BuildIndex(const std::string &location, EngineType engine_type) = 0;
virtual Status Cache() = 0;
@ -89,7 +89,6 @@ public:
using ExecutionEnginePtr = std::shared_ptr<ExecutionEngine>;
} // namespace engine
} // namespace milvus
} // namespace zilliz

87
cpp/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "ExecutionEngineImpl.h"
#include "db/engine/ExecutionEngineImpl.h"
#include "cache/GpuCacheMgr.h"
#include "cache/CpuCacheMgr.h"
#include "metrics/Metrics.h"
@ -29,6 +29,7 @@
#include "server/Config.h"
#include <stdexcept>
#include <utility>
namespace zilliz {
namespace milvus {
@ -72,7 +73,8 @@ ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index,
nlist_(nlist) {
}
VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
VecIndexPtr
ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
std::shared_ptr<VecIndex> index;
switch (type) {
case EngineType::FAISS_IDMAP: {
@ -99,41 +101,48 @@ VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
return index;
}
Status ExecutionEngineImpl::AddWithIds(long n, const float *xdata, const long *xids) {
Status
ExecutionEngineImpl::AddWithIds(int64_t n, const float *xdata, const int64_t *xids) {
auto status = index_->Add(n, xdata, xids);
return status;
}
size_t ExecutionEngineImpl::Count() const {
if(index_ == nullptr) {
size_t
ExecutionEngineImpl::Count() const {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return count 0";
return 0;
}
return index_->Count();
}
size_t ExecutionEngineImpl::Size() const {
size_t
ExecutionEngineImpl::Size() const {
return (size_t) (Count() * Dimension()) * sizeof(float);
}
size_t ExecutionEngineImpl::Dimension() const {
if(index_ == nullptr) {
size_t
ExecutionEngineImpl::Dimension() const {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return dimension " << dim_;
return dim_;
}
return index_->Dimension();
}
size_t ExecutionEngineImpl::PhysicalSize() const {
size_t
ExecutionEngineImpl::PhysicalSize() const {
return server::CommonUtil::GetFileSize(location_);
}
Status ExecutionEngineImpl::Serialize() {
Status
ExecutionEngineImpl::Serialize() {
auto status = write_index(index_, location_);
return status;
}
Status ExecutionEngineImpl::Load(bool to_cache) {
Status
ExecutionEngineImpl::Load(bool to_cache) {
index_ = cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
bool already_in_cache = (index_ != nullptr);
if (!already_in_cache) {
@ -141,7 +150,7 @@ Status ExecutionEngineImpl::Load(bool to_cache) {
double physical_size = PhysicalSize();
server::CollectExecutionEngineMetrics metrics(physical_size);
index_ = read_index(location_);
if(index_ == nullptr) {
if (index_ == nullptr) {
std::string msg = "Failed to load index from " + location_;
ENGINE_LOG_ERROR << msg;
return Status(DB_ERROR, msg);
@ -160,13 +169,14 @@ Status ExecutionEngineImpl::Load(bool to_cache) {
return Status::OK();
}
Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
Status
ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
auto index = cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_);
bool already_in_cache = (index != nullptr);
if (already_in_cache) {
index_ = index;
} else {
if(index_ == nullptr) {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to gpu";
return Status(DB_ERROR, "index is null");
}
@ -187,13 +197,14 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
return Status::OK();
}
Status ExecutionEngineImpl::CopyToCpu() {
Status
ExecutionEngineImpl::CopyToCpu() {
auto index = cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
bool already_in_cache = (index != nullptr);
if (already_in_cache) {
index_ = index;
} else {
if(index_ == nullptr) {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to cpu";
return Status(DB_ERROR, "index is null");
}
@ -213,8 +224,9 @@ Status ExecutionEngineImpl::CopyToCpu() {
return Status::OK();
}
ExecutionEnginePtr ExecutionEngineImpl::Clone() {
if(index_ == nullptr) {
ExecutionEnginePtr
ExecutionEngineImpl::Clone() {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to clone";
return nullptr;
}
@ -225,7 +237,8 @@ ExecutionEnginePtr ExecutionEngineImpl::Clone() {
return ret;
}
Status ExecutionEngineImpl::Merge(const std::string &location) {
Status
ExecutionEngineImpl::Merge(const std::string &location) {
if (location == location_) {
return Status(DB_ERROR, "Cannot Merge Self");
}
@ -243,7 +256,7 @@ Status ExecutionEngineImpl::Merge(const std::string &location) {
}
}
if(index_ == nullptr) {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to merge";
return Status(DB_ERROR, "index is null");
}
@ -264,7 +277,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_;
auto from_index = std::dynamic_pointer_cast<BFIndex>(index_);
if(from_index == nullptr) {
if (from_index == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: from_index is null, failed to build index";
return nullptr;
}
@ -282,21 +295,22 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
AutoGenParams(to_index->GetType(), Count(), build_cfg);
auto status = to_index->BuildAll(Count(),
from_index->GetRawVectors(),
from_index->GetRawIds(),
build_cfg);
from_index->GetRawVectors(),
from_index->GetRawIds(),
build_cfg);
if (!status.ok()) { throw Exception(DB_ERROR, status.message()); }
return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_);
}
Status ExecutionEngineImpl::Search(long n,
const float *data,
long k,
long nprobe,
float *distances,
long *labels) const {
if(index_ == nullptr) {
Status
ExecutionEngineImpl::Search(int64_t n,
const float *data,
int64_t k,
int64_t nprobe,
float *distances,
int64_t *labels) const {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search";
return Status(DB_ERROR, "index is null");
}
@ -310,14 +324,16 @@ Status ExecutionEngineImpl::Search(long n,
return status;
}
Status ExecutionEngineImpl::Cache() {
Status
ExecutionEngineImpl::Cache() {
cache::DataObjPtr obj = std::make_shared<cache::DataObj>(index_, PhysicalSize());
zilliz::milvus::cache::CpuCacheMgr::GetInstance()->InsertItem(location_, obj);
return Status::OK();
}
Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
Status
ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
cache::DataObjPtr obj = std::make_shared<cache::DataObj>(index_, PhysicalSize());
zilliz::milvus::cache::GpuCacheMgr::GetInstance(gpu_id)->InsertItem(location_, obj);
@ -325,8 +341,8 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
}
// TODO(linxj): remove.
Status ExecutionEngineImpl::Init() {
using namespace zilliz::milvus::server;
Status
ExecutionEngineImpl::Init() {
server::Config &config = server::Config::GetInstance();
Status s = config.GetDBConfigBuildIndexGPU(gpu_num_);
if (!s.ok()) return s;
@ -334,7 +350,6 @@ Status ExecutionEngineImpl::Init() {
return Status::OK();
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

32
cpp/src/db/engine/ExecutionEngineImpl.h
View File

@ -23,15 +23,12 @@
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
class ExecutionEngineImpl : public ExecutionEngine {
public:
public:
ExecutionEngineImpl(uint16_t dimension,
const std::string &location,
EngineType index_type,
@ -44,7 +41,7 @@ public:
MetricType metric_type,
int32_t nlist);
Status AddWithIds(long n, const float *xdata, const long *xids) override;
Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) override;
size_t Count() const override;
@ -66,12 +63,12 @@ public:
Status Merge(const std::string &location) override;
Status Search(long n,
Status Search(int64_t n,
const float *data,
long k,
long nprobe,
int64_t k,
int64_t nprobe,
float *distances,
long *labels) const override;
int64_t *labels) const override;
ExecutionEnginePtr BuildIndex(const std::string &location, EngineType engine_type) override;
@ -81,18 +78,24 @@ public:
Status Init() override;
EngineType IndexEngineType() const override { return index_type_; }
EngineType IndexEngineType() const override {
return index_type_;
}
MetricType IndexMetricType() const override { return metric_type_; }
MetricType IndexMetricType() const override {
return metric_type_;
}
std::string GetLocation() const override { return location_; }
std::string GetLocation() const override {
return location_;
}
private:
private:
VecIndexPtr CreatetVecIndex(EngineType type);
VecIndexPtr Load(const std::string &location);
protected:
protected:
VecIndexPtr index_ = nullptr;
EngineType index_type_;
MetricType metric_type_;
@ -104,7 +107,6 @@ protected:
int32_t gpu_num_ = 0;
};
} // namespace engine
} // namespace milvus
} // namespace zilliz

5
cpp/src/db/insert/MemManager.h
View File

@ -23,6 +23,7 @@
#include <set>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
@ -30,7 +31,6 @@ namespace engine {
class MemManager {
public:
virtual Status InsertVectors(const std::string &table_id,
size_t n, const float *vectors, IDNumbers &vector_ids) = 0;
@ -43,11 +43,10 @@ class MemManager {
virtual size_t GetCurrentImmutableMem() = 0;
virtual size_t GetCurrentMem() = 0;
}; // MemManagerAbstract
using MemManagerPtr = std::shared_ptr<MemManager>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

48
cpp/src/db/insert/MemManagerImpl.cpp
View File

@ -16,19 +16,19 @@
// under the License.
#include "MemManagerImpl.h"
#include "db/insert/MemManagerImpl.h"
#include "VectorSource.h"
#include "utils/Log.h"
#include "db/Constants.h"
#include <thread>
namespace zilliz {
namespace milvus {
namespace engine {
MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) {
MemTablePtr
MemManagerImpl::GetMemByTable(const std::string &table_id) {
auto memIt = mem_id_map_.find(table_id);
if (memIt != mem_id_map_.end()) {
return memIt->second;
@ -38,11 +38,11 @@ MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) {
return mem_id_map_[table_id];
}
Status MemManagerImpl::InsertVectors(const std::string &table_id_,
size_t n_,
const float *vectors_,
IDNumbers &vector_ids_) {
Status
MemManagerImpl::InsertVectors(const std::string &table_id_,
size_t n_,
const float *vectors_,
IDNumbers &vector_ids_) {
while (GetCurrentMem() > options_.insert_buffer_size_) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
@ -52,11 +52,11 @@ Status MemManagerImpl::InsertVectors(const std::string &table_id_,
return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_);
}
Status MemManagerImpl::InsertVectorsNoLock(const std::string &table_id,
size_t n,
const float *vectors,
IDNumbers &vector_ids) {
Status
MemManagerImpl::InsertVectorsNoLock(const std::string &table_id,
size_t n,
const float *vectors,
IDNumbers &vector_ids) {
MemTablePtr mem = GetMemByTable(table_id);
VectorSourcePtr source = std::make_shared<VectorSource>(n, vectors);
@ -69,10 +69,11 @@ Status MemManagerImpl::InsertVectorsNoLock(const std::string &table_id,
return status;
}
Status MemManagerImpl::ToImmutable() {
Status
MemManagerImpl::ToImmutable() {
std::unique_lock<std::mutex> lock(mutex_);
MemIdMap temp_map;
for (auto &kv: mem_id_map_) {
for (auto &kv : mem_id_map_) {
if (kv.second->Empty()) {
//empty table, no need to serialize
temp_map.insert(kv);
@ -85,7 +86,8 @@ Status MemManagerImpl::ToImmutable() {
return Status::OK();
}
Status MemManagerImpl::Serialize(std::set<std::string> &table_ids) {
Status
MemManagerImpl::Serialize(std::set<std::string> &table_ids) {
ToImmutable();
std::unique_lock<std::mutex> lock(serialization_mtx_);
table_ids.clear();
@ -97,7 +99,8 @@ Status MemManagerImpl::Serialize(std::set<std::string> &table_ids) {
return Status::OK();
}
Status MemManagerImpl::EraseMemVector(const std::string &table_id) {
Status
MemManagerImpl::EraseMemVector(const std::string &table_id) {
{//erase MemVector from rapid-insert cache
std::unique_lock<std::mutex> lock(mutex_);
mem_id_map_.erase(table_id);
@ -117,7 +120,8 @@ Status MemManagerImpl::EraseMemVector(const std::string &table_id) {
return Status::OK();
}
size_t MemManagerImpl::GetCurrentMutableMem() {
size_t
MemManagerImpl::GetCurrentMutableMem() {
size_t total_mem = 0;
for (auto &kv : mem_id_map_) {
auto memTable = kv.second;
@ -126,7 +130,8 @@ size_t MemManagerImpl::GetCurrentMutableMem() {
return total_mem;
}
size_t MemManagerImpl::GetCurrentImmutableMem() {
size_t
MemManagerImpl::GetCurrentImmutableMem() {
size_t total_mem = 0;
for (auto &mem_table : immu_mem_list_) {
total_mem += mem_table->GetCurrentMem();
@ -134,10 +139,11 @@ size_t MemManagerImpl::GetCurrentImmutableMem() {
return total_mem;
}
size_t MemManagerImpl::GetCurrentMem() {
size_t
MemManagerImpl::GetCurrentMem() {
return GetCurrentMutableMem() + GetCurrentImmutableMem();
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

9
cpp/src/db/insert/MemManagerImpl.h
View File

@ -24,12 +24,13 @@
#include "utils/Status.h"
#include <map>
#include <set>
#include <vector>
#include <string>
#include <ctime>
#include <memory>
#include <mutex>
namespace zilliz {
namespace milvus {
namespace engine {
@ -39,7 +40,8 @@ class MemManagerImpl : public MemManager {
using Ptr = std::shared_ptr<MemManagerImpl>;
MemManagerImpl(const meta::MetaPtr &meta, const DBOptions &options)
: meta_(meta), options_(options) {}
: meta_(meta), options_(options) {
}
Status InsertVectors(const std::string &table_id,
size_t n, const float *vectors, IDNumbers &vector_ids) override;
@ -71,7 +73,6 @@ class MemManagerImpl : public MemManager {
std::mutex serialization_mtx_;
}; // NewMemManager
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

6
cpp/src/db/insert/MemMenagerFactory.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "MemMenagerFactory.h"
#include "db/insert/MemMenagerFactory.h"
#include "MemManagerImpl.h"
#include "utils/Log.h"
#include "utils/Exception.h"
@ -26,12 +26,14 @@
#include <cstdlib>
#include <string>
#include <regex>
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
MemManagerPtr MemManagerFactory::Build(const std::shared_ptr<meta::Meta>& meta, const DBOptions& options) {
MemManagerPtr
MemManagerFactory::Build(const std::shared_ptr<meta::Meta> &meta, const DBOptions &options) {
return std::make_shared<MemManagerImpl>(meta, options);
}

10
cpp/src/db/insert/MemMenagerFactory.h
View File

@ -20,15 +20,17 @@
#include "MemManager.h"
#include "db/meta/Meta.h"
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
class MemManagerFactory {
public:
public:
static MemManagerPtr Build(const std::shared_ptr<meta::Meta> &meta, const DBOptions &options);
};
}
}
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

30
cpp/src/db/insert/MemTable.cpp
View File

@ -16,9 +16,11 @@
// under the License.
#include "MemTable.h"
#include "db/insert/MemTable.h"
#include "utils/Log.h"
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
@ -30,13 +32,11 @@ MemTable::MemTable(const std::string &table_id,
table_id_(table_id),
meta_(meta),
options_(options) {
}
Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
Status
MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
while (!source->AllAdded()) {
MemTableFilePtr current_mem_table_file;
if (!mem_table_file_list_.empty()) {
current_mem_table_file = mem_table_file_list_.back();
@ -62,15 +62,18 @@ Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
return Status::OK();
}
void MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) {
void
MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) {
mem_table_file = mem_table_file_list_.back();
}
size_t MemTable::GetTableFileCount() {
size_t
MemTable::GetTableFileCount() {
return mem_table_file_list_.size();
}
Status MemTable::Serialize() {
Status
MemTable::Serialize() {
for (auto mem_table_file = mem_table_file_list_.begin(); mem_table_file != mem_table_file_list_.end();) {
auto status = (*mem_table_file)->Serialize();
if (!status.ok()) {
@ -84,15 +87,18 @@ Status MemTable::Serialize() {
return Status::OK();
}
bool MemTable::Empty() {
bool
MemTable::Empty() {
return mem_table_file_list_.empty();
}
const std::string &MemTable::GetTableId() const {
const std::string &
MemTable::GetTableId() const {
return table_id_;
}
size_t MemTable::GetCurrentMem() {
size_t
MemTable::GetCurrentMem() {
std::lock_guard<std::mutex> lock(mutex_);
size_t total_mem = 0;
for (auto &mem_table_file : mem_table_file_list_) {
@ -103,4 +109,4 @@ size_t MemTable::GetCurrentMem() {
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

7
cpp/src/db/insert/MemTable.h
View File

@ -23,7 +23,9 @@
#include "utils/Status.h"
#include <mutex>
#include <vector>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
@ -59,11 +61,10 @@ class MemTable {
DBOptions options_;
std::mutex mutex_;
}; //MemTable
using MemTablePtr = std::shared_ptr<MemTable>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

38
cpp/src/db/insert/MemTableFile.cpp
View File

@ -16,14 +16,14 @@
// under the License.
#include "MemTableFile.h"
#include "db/insert/MemTableFile.h"
#include "db/Constants.h"
#include "db/engine/EngineFactory.h"
#include "metrics/Metrics.h"
#include "utils/Log.h"
#include <cmath>
#include <string>
namespace zilliz {
namespace milvus {
@ -35,20 +35,19 @@ MemTableFile::MemTableFile(const std::string &table_id,
table_id_(table_id),
meta_(meta),
options_(options) {
current_mem_ = 0;
auto status = CreateTableFile();
if (status.ok()) {
execution_engine_ = EngineFactory::Build(table_file_schema_.dimension_,
table_file_schema_.location_,
(EngineType) table_file_schema_.engine_type_,
(MetricType)table_file_schema_.metric_type_,
(MetricType) table_file_schema_.metric_type_,
table_file_schema_.nlist_);
}
}
Status MemTableFile::CreateTableFile() {
Status
MemTableFile::CreateTableFile() {
meta::TableFileSchema table_file_schema;
table_file_schema.table_id_ = table_id_;
auto status = meta_->CreateTableFile(table_file_schema);
@ -61,8 +60,8 @@ Status MemTableFile::CreateTableFile() {
return status;
}
Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) {
Status
MemTableFile::Add(const VectorSourcePtr &source, IDNumbers &vector_ids) {
if (table_file_schema_.dimension_ <= 0) {
std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " +
std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_;
@ -75,7 +74,8 @@ Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) {
if (mem_left >= single_vector_mem_size) {
size_t num_vectors_to_add = std::ceil(mem_left / single_vector_mem_size);
size_t num_vectors_added;
auto status = source->Add(execution_engine_, table_file_schema_, num_vectors_to_add, num_vectors_added, vector_ids);
auto status =
source->Add(execution_engine_, table_file_schema_, num_vectors_to_add, num_vectors_added, vector_ids);
if (status.ok()) {
current_mem_ += (num_vectors_added * single_vector_mem_size);
}
@ -84,20 +84,24 @@ Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) {
return Status::OK();
}
size_t MemTableFile::GetCurrentMem() {
size_t
MemTableFile::GetCurrentMem() {
return current_mem_;
}
size_t MemTableFile::GetMemLeft() {
size_t
MemTableFile::GetMemLeft() {
return (MAX_TABLE_FILE_MEM - current_mem_);
}
bool MemTableFile::IsFull() {
bool
MemTableFile::IsFull() {
size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE;
return (GetMemLeft() < single_vector_mem_size);
}
Status MemTableFile::Serialize() {
Status
MemTableFile::Serialize() {
size_t size = GetCurrentMem();
server::CollectSerializeMetrics metrics(size);
@ -107,7 +111,7 @@ Status MemTableFile::Serialize() {
//if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
//else set file type to RAW, no need to build index
if (table_file_schema_.engine_type_ != (int)EngineType::FAISS_IDMAP) {
if (table_file_schema_.engine_type_ != (int) EngineType::FAISS_IDMAP) {
table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
} else {
@ -117,9 +121,9 @@ Status MemTableFile::Serialize() {
auto status = meta_->UpdateTableFile(table_file_schema_);
ENGINE_LOG_DEBUG << "New " << ((table_file_schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index")
<< " file " << table_file_schema_.file_id_ << " of size " << size << " bytes";
<< " file " << table_file_schema_.file_id_ << " of size " << size << " bytes";
if(options_.insert_cache_immediately_) {
if (options_.insert_cache_immediately_) {
execution_engine_->Cache();
}
@ -128,4 +132,4 @@ Status MemTableFile::Serialize() {
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

14
cpp/src/db/insert/MemTableFile.h
View File

@ -23,17 +23,18 @@
#include "db/engine/ExecutionEngine.h"
#include "utils/Status.h"
#include <string>
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
class MemTableFile {
public:
MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options);
Status Add(const VectorSourcePtr &source, IDNumbers& vector_ids);
Status Add(const VectorSourcePtr &source, IDNumbers &vector_ids);
size_t GetCurrentMem();
@ -44,25 +45,20 @@ class MemTableFile {
Status Serialize();
private:
Status CreateTableFile();
private:
const std::string table_id_;
meta::TableFileSchema table_file_schema_;
meta::MetaPtr meta_;
DBOptions options_;
size_t current_mem_;
ExecutionEnginePtr execution_engine_;
}; //MemTableFile
using MemTableFilePtr = std::shared_ptr<MemTableFile>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

35
cpp/src/db/insert/VectorSource.cpp
View File

@ -16,32 +16,30 @@
// under the License.
#include "VectorSource.h"
#include "db/insert/VectorSource.h"
#include "db/engine/ExecutionEngine.h"
#include "db/engine/EngineFactory.h"
#include "utils/Log.h"
#include "metrics/Metrics.h"
namespace zilliz {
namespace milvus {
namespace engine {
VectorSource::VectorSource(const size_t &n,
const float *vectors) :
n_(n),
vectors_(vectors),
id_generator_(std::make_shared<SimpleIDGenerator>()) {
n_(n),
vectors_(vectors),
id_generator_(std::make_shared<SimpleIDGenerator>()) {
current_num_vectors_added = 0;
}
Status VectorSource::Add(const ExecutionEnginePtr &execution_engine,
const meta::TableFileSchema &table_file_schema,
const size_t &num_vectors_to_add,
size_t &num_vectors_added,
IDNumbers &vector_ids) {
Status
VectorSource::Add(const ExecutionEnginePtr &execution_engine,
const meta::TableFileSchema &table_file_schema,
const size_t &num_vectors_to_add,
size_t &num_vectors_added,
IDNumbers &vector_ids) {
server::CollectAddMetrics metrics(n_, table_file_schema.dimension_);
num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ?
@ -52,7 +50,7 @@ Status VectorSource::Add(const ExecutionEnginePtr &execution_engine,
} else {
vector_ids_to_add.resize(num_vectors_added);
for (int pos = current_num_vectors_added; pos < current_num_vectors_added + num_vectors_added; pos++) {
vector_ids_to_add[pos-current_num_vectors_added] = vector_ids[pos];
vector_ids_to_add[pos - current_num_vectors_added] = vector_ids[pos];
}
}
Status status = execution_engine->AddWithIds(num_vectors_added,
@ -70,18 +68,21 @@ Status VectorSource::Add(const ExecutionEnginePtr &execution_engine,
return status;
}
size_t VectorSource::GetNumVectorsAdded() {
size_t
VectorSource::GetNumVectorsAdded() {
return current_num_vectors_added;
}
bool VectorSource::AllAdded() {
bool
VectorSource::AllAdded() {
return (current_num_vectors_added == n_);
}
IDNumbers VectorSource::GetVectorIds() {
IDNumbers
VectorSource::GetVectorIds() {
return vector_ids_;
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

5
cpp/src/db/insert/VectorSource.h
View File

@ -23,6 +23,7 @@
#include "db/engine/ExecutionEngine.h"
#include "utils/Status.h"
#include <memory>
namespace zilliz {
namespace milvus {
@ -45,7 +46,6 @@ class VectorSource {
IDNumbers GetVectorIds();
private:
const size_t n_;
const float *vectors_;
IDNumbers vector_ids_;
@ -53,11 +53,10 @@ class VectorSource {
size_t current_num_vectors_added;
std::shared_ptr<IDGenerator> id_generator_;
}; //VectorSource
using VectorSourcePtr = std::shared_ptr<VectorSource>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

13
cpp/src/db/meta/Meta.h
View File

@ -25,14 +25,16 @@
#include <cstddef>
#include <memory>
#include <vector>
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
namespace meta {
static const char* META_TABLES = "Tables";
static const char* META_TABLEFILES = "TableFiles";
static const char *META_TABLES = "Tables";
static const char *META_TABLEFILES = "TableFiles";
class Meta {
public:
@ -46,7 +48,7 @@ class Meta {
virtual Status AllTables(std::vector<TableSchema> &table_schema_array) = 0;
virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) = 0;
virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) = 0;
virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0;
@ -83,9 +85,9 @@ class Meta {
virtual Status FilesByType(const std::string &table_id,
const std::vector<int> &file_types,
std::vector<std::string>& file_ids) = 0;
std::vector<std::string> &file_ids) = 0;
virtual Status DescribeTableIndex(const std::string &table_id, TableIndex& index) = 0;
virtual Status DescribeTableIndex(const std::string &table_id, TableIndex &index) = 0;
virtual Status DropTableIndex(const std::string &table_id) = 0;
@ -96,7 +98,6 @@ class Meta {
virtual Status DropAll() = 0;
virtual Status Count(const std::string &table_id, uint64_t &result) = 0;
}; // MetaData
using MetaPtr = std::shared_ptr<Meta>;

20
cpp/src/db/meta/MetaConsts.h
View File

@ -23,20 +23,20 @@ namespace engine {
namespace meta {
const size_t K = 1024UL;
const size_t M = K*K;
const size_t G = K*M;
const size_t T = K*G;
const size_t M = K * K;
const size_t G = K * M;
const size_t T = K * G;
const size_t S_PS = 1UL;
const size_t MS_PS = 1000*S_PS;
const size_t US_PS = 1000*MS_PS;
const size_t NS_PS = 1000*US_PS;
const size_t MS_PS = 1000 * S_PS;
const size_t US_PS = 1000 * MS_PS;
const size_t NS_PS = 1000 * US_PS;
const size_t SECOND = 1UL;
const size_t M_SEC = 60*SECOND;
const size_t H_SEC = 60*M_SEC;
const size_t D_SEC = 24*H_SEC;
const size_t W_SEC = 7*D_SEC;
const size_t M_SEC = 60 * SECOND;
const size_t H_SEC = 60 * M_SEC;
const size_t D_SEC = 24 * H_SEC;
const size_t W_SEC = 7 * D_SEC;
} // namespace meta
} // namespace engine

67
cpp/src/db/meta/MetaFactory.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "MetaFactory.h"
#include "db/meta/MetaFactory.h"
#include "SqliteMetaImpl.h"
#include "MySQLMetaImpl.h"
#include "utils/Log.h"
@ -28,47 +28,52 @@
#include <cstdlib>
#include <string>
#include <string.h>
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
DBMetaOptions MetaFactory::BuildOption(const std::string &path) {
auto p = path;
if(p == "") {
srand(time(nullptr));
std::stringstream ss;
ss << "/tmp/" << rand();
p = ss.str();
}
DBMetaOptions meta;
meta.path_ = p;
return meta;
DBMetaOptions
MetaFactory::BuildOption(const std::string &path) {
auto p = path;
if (p == "") {
srand(time(nullptr));
std::stringstream ss;
uint32_t rd = 0;
rand_r(&rd);
ss << "/tmp/" << rd;
p = ss.str();
}
meta::MetaPtr MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) {
std::string uri = metaOptions.backend_uri_;
DBMetaOptions meta;
meta.path_ = p;
return meta;
}
utils::MetaUriInfo uri_info;
auto status = utils::ParseMetaUri(uri, uri_info);
if(!status.ok()) {
ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri;
throw InvalidArgumentException("Wrong URI format ");
}
meta::MetaPtr
MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) {
std::string uri = metaOptions.backend_uri_;
if (strcasecmp(uri_info.dialect_.c_str(), "mysql") == 0) {
ENGINE_LOG_INFO << "Using MySQL";
return std::make_shared<meta::MySQLMetaImpl>(metaOptions, mode);
} else if (strcasecmp(uri_info.dialect_.c_str(), "sqlite") == 0) {
ENGINE_LOG_INFO << "Using SQLite";
return std::make_shared<meta::SqliteMetaImpl>(metaOptions);
} else {
ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << uri_info.dialect_;
throw InvalidArgumentException("URI dialect is not mysql / sqlite");
}
utils::MetaUriInfo uri_info;
auto status = utils::ParseMetaUri(uri, uri_info);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri;
throw InvalidArgumentException("Wrong URI format ");
}
if (strcasecmp(uri_info.dialect_.c_str(), "mysql") == 0) {
ENGINE_LOG_INFO << "Using MySQL";
return std::make_shared<meta::MySQLMetaImpl>(metaOptions, mode);
} else if (strcasecmp(uri_info.dialect_.c_str(), "sqlite") == 0) {
ENGINE_LOG_INFO << "Using SQLite";
return std::make_shared<meta::SqliteMetaImpl>(metaOptions);
} else {
ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << uri_info.dialect_;
throw InvalidArgumentException("URI dialect is not mysql / sqlite");
}
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

11
cpp/src/db/meta/MetaFactory.h
View File

@ -20,18 +20,19 @@
#include "Meta.h"
#include "db/Options.h"
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
class MetaFactory {
public:
public:
static DBMetaOptions BuildOption(const std::string &path = "");
static meta::MetaPtr Build(const DBMetaOptions &metaOptions, const int &mode);
};
}
}
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

11
cpp/src/db/meta/MetaTypes.h
View File

@ -23,21 +23,22 @@
#include <vector>
#include <map>
#include <string>
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
namespace meta {
constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP;
constexpr int32_t DEFAULT_ENGINE_TYPE = (int) EngineType::FAISS_IDMAP;
constexpr int32_t DEFAULT_NLIST = 16384;
constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2;
constexpr int32_t DEFAULT_METRIC_TYPE = (int) MetricType::L2;
constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB;
constexpr int64_t FLAG_MASK_NO_USERID = 0x1;
constexpr int64_t FLAG_MASK_HAS_USERID = 0x1<<1;
constexpr int64_t FLAG_MASK_HAS_USERID = 0x1 << 1;
using DateT = int ;
using DateT = int;
const DateT EmptyDate = -1;
using DatesT = std::vector<DateT>;
@ -49,7 +50,7 @@ struct TableSchema {
size_t id_ = 0;
std::string table_id_;
int32_t state_ = (int)NORMAL;
int32_t state_ = (int) NORMAL;
uint16_t dimension_ = 0;
int64_t created_on_ = 0;
int64_t flag_ = 0;

105
cpp/src/db/meta/MySQLConnectionPool.cpp
View File

@ -16,37 +16,39 @@
// under the License.
#include "MySQLConnectionPool.h"
#include "db/meta/MySQLConnectionPool.h"
namespace zilliz {
namespace milvus {
namespace engine {
namespace meta {
// Do a simple form of in-use connection limiting: wait to return
// a connection until there are a reasonably low number in use
// already. Can't do this in create() because we're interested in
// connections actually in use, not those created. Also note that
// we keep our own count; ConnectionPool::size() isn't the same!
mysqlpp::Connection *MySQLConnectionPool::grab() {
while (conns_in_use_ > max_pool_size_) {
sleep(1);
}
++conns_in_use_;
return mysqlpp::ConnectionPool::grab();
// Do a simple form of in-use connection limiting: wait to return
// a connection until there are a reasonably low number in use
// already. Can't do this in create() because we're interested in
// connections actually in use, not those created. Also note that
// we keep our own count; ConnectionPool::size() isn't the same!
mysqlpp::Connection *
MySQLConnectionPool::grab() {
while (conns_in_use_ > max_pool_size_) {
sleep(1);
}
// Other half of in-use conn count limit
void MySQLConnectionPool::release(const mysqlpp::Connection *pc) {
mysqlpp::ConnectionPool::release(pc);
++conns_in_use_;
return mysqlpp::ConnectionPool::grab();
}
if (conns_in_use_ <= 0) {
ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " << conns_in_use_;
} else {
--conns_in_use_;
}
// Other half of in-use conn count limit
void
MySQLConnectionPool::release(const mysqlpp::Connection *pc) {
mysqlpp::ConnectionPool::release(pc);
if (conns_in_use_ <= 0) {
ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = "
<< conns_in_use_;
} else {
--conns_in_use_;
}
}
// int MySQLConnectionPool::getConnectionsInUse() {
// return conns_in_use_;
@ -56,39 +58,42 @@ namespace meta {
// max_idle_time_ = max_idle;
// }
std::string MySQLConnectionPool::getDB() {
return db_;
}
std::string
MySQLConnectionPool::getDB() {
return db_;
}
// Superclass overrides
mysqlpp::Connection *MySQLConnectionPool::create() {
try {
// Create connection using the parameters we were passed upon
// creation.
mysqlpp::Connection *conn = new mysqlpp::Connection();
conn->set_option(new mysqlpp::ReconnectOption(true));
conn->connect(db_.empty() ? 0 : db_.c_str(),
server_.empty() ? 0 : server_.c_str(),
user_.empty() ? 0 : user_.c_str(),
password_.empty() ? 0 : password_.c_str(),
port_);
return conn;
} catch (const mysqlpp::ConnectionFailed& er) {
ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what();
return nullptr;
}
// Superclass overrides
mysqlpp::Connection *
MySQLConnectionPool::create() {
try {
// Create connection using the parameters we were passed upon
// creation.
mysqlpp::Connection *conn = new mysqlpp::Connection();
conn->set_option(new mysqlpp::ReconnectOption(true));
conn->connect(db_.empty() ? 0 : db_.c_str(),
server_.empty() ? 0 : server_.c_str(),
user_.empty() ? 0 : user_.c_str(),
password_.empty() ? 0 : password_.c_str(),
port_);
return conn;
} catch (const mysqlpp::ConnectionFailed &er) {
ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what();
return nullptr;
}
}
void MySQLConnectionPool::destroy(mysqlpp::Connection *cp) {
// Our superclass can't know how we created the Connection, so
// it delegates destruction to us, to be safe.
delete cp;
}
void
MySQLConnectionPool::destroy(mysqlpp::Connection *cp) {
// Our superclass can't know how we created the Connection, so
// it delegates destruction to us, to be safe.
delete cp;
}
unsigned int MySQLConnectionPool::max_idle_time() {
return max_idle_time_;
}
unsigned int
MySQLConnectionPool::max_idle_time() {
return max_idle_time_;
}
} // namespace meta
} // namespace engine

24
cpp/src/db/meta/MySQLConnectionPool.h
View File

@ -30,8 +30,7 @@ namespace engine {
namespace meta {
class MySQLConnectionPool : public mysqlpp::ConnectionPool {
public:
public:
// The object's only constructor
MySQLConnectionPool(std::string dbName,
std::string userName,
@ -39,15 +38,13 @@ public:
std::string serverIp,
int port = 0,
int maxPoolSize = 8) :
db_(dbName),
user_(userName),
password_(passWord),
server_(serverIp),
port_(port),
max_pool_size_(maxPoolSize) {
db_(dbName),
user_(userName),
password_(passWord),
server_(serverIp),
port_(port),
max_pool_size_(maxPoolSize) {
conns_in_use_ = 0;
max_idle_time_ = 10; //10 seconds
}
@ -68,8 +65,7 @@ public:
std::string getDB();
protected:
protected:
// Superclass overrides
mysqlpp::Connection *create() override;
@ -77,7 +73,7 @@ protected:
unsigned int max_idle_time() override;
private:
private:
// Number of connections currently in use
std::atomic<int> conns_in_use_;
@ -93,4 +89,4 @@ private:
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

594
cpp/src/db/meta/MySQLMetaImpl.cpp
View File

File diff suppressed because it is too large Load Diff

21
cpp/src/db/meta/MySQLMetaImpl.h
View File

@ -21,26 +21,25 @@
#include "db/Options.h"
#include "MySQLConnectionPool.h"
#include "mysql++/mysql++.h"
#include <mysql++/mysql++.h>
#include <mutex>
#include <vector>
#include <string>
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
namespace meta {
// auto StoragePrototype(const std::string& path);
using namespace mysqlpp;
class MySQLMetaImpl : public Meta {
public:
MySQLMetaImpl(const DBMetaOptions &options_, const int &mode);
MySQLMetaImpl(const DBMetaOptions &options, const int &mode);
~MySQLMetaImpl();
Status CreateTable(TableSchema &table_schema) override;
Status DescribeTable(TableSchema &group_info_) override;
Status DescribeTable(TableSchema &table_schema) override;
Status HasTable(const std::string &table_id, bool &has_or_not) override;
@ -63,11 +62,11 @@ class MySQLMetaImpl : public Meta {
const std::vector<int> &file_types,
std::vector<std::string> &file_ids) override;
Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) override;
Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override;
Status UpdateTableFlag(const std::string &table_id, int64_t flag) override;
Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override;
Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override;
Status DropTableIndex(const std::string &table_id) override;
@ -102,12 +101,12 @@ class MySQLMetaImpl : public Meta {
private:
Status NextFileId(std::string &file_id);
Status NextTableId(std::string &table_id);
Status DiscardFiles(long long to_discard_size);
Status DiscardFiles(int64_t to_discard_size);
void ValidateMetaSchema();
Status Initialize();
private:
private:
const DBMetaOptions options_;
const int mode_;

438
cpp/src/db/meta/SqliteMetaImpl.cpp
View File

File diff suppressed because it is too large Load Diff

17
cpp/src/db/meta/SqliteMetaImpl.h
View File

@ -21,22 +21,25 @@
#include "db/Options.h"
#include <mutex>
#include <vector>
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
namespace meta {
auto StoragePrototype(const std::string &path);
auto
StoragePrototype(const std::string &path);
class SqliteMetaImpl : public Meta {
public:
explicit SqliteMetaImpl(const DBMetaOptions &options_);
explicit SqliteMetaImpl(const DBMetaOptions &options);
~SqliteMetaImpl();
Status CreateTable(TableSchema &table_schema) override;
Status DescribeTable(TableSchema &group_info_) override;
Status DescribeTable(TableSchema &table_schema) override;
Status HasTable(const std::string &table_id, bool &has_or_not) override;
@ -58,11 +61,11 @@ class SqliteMetaImpl : public Meta {
const std::vector<int> &file_types,
std::vector<std::string> &file_ids) override;
Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) override;
Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override;
Status UpdateTableFlag(const std::string &table_id, int64_t flag) override;
Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override;
Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override;
Status DropTableIndex(const std::string &table_id) override;
@ -96,12 +99,12 @@ class SqliteMetaImpl : public Meta {
private:
Status NextFileId(std::string &file_id);
Status NextTableId(std::string &table_id);
Status DiscardFiles(long to_discard_size);
Status DiscardFiles(int64_t to_discard_size);
void ValidateMetaSchema();
Status Initialize();
private:
private:
const DBMetaOptions options_;
std::mutex meta_mutex_;
}; // DBMetaImpl

20
cpp/src/main.cpp
View File

@ -27,15 +27,13 @@
#include "utils/CommonUtil.h"
#include "metrics/Metrics.h"
#include "server/Server.h"
#include "version.h"
#include "../version.h"
INITIALIZE_EASYLOGGINGPP
void print_help(const std::string &app_name);
using namespace zilliz::milvus;
int
main(int argc, char *argv[]) {
std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl;
@ -78,7 +76,6 @@ main(int argc, char *argv[]) {
std::cout << "Initial log config from: " << log_config_file << std::endl;
break;
}
case 'p': {
char *pid_filename_ptr = strdup(optarg);
pid_filename = pid_filename_ptr;
@ -86,7 +83,6 @@ main(int argc, char *argv[]) {
std::cout << pid_filename << std::endl;
break;
}
case 'd':
start_daemonized = 1;
break;
@ -103,14 +99,14 @@ main(int argc, char *argv[]) {
}
/* Handle Signal */
signal(SIGHUP, server::SignalUtil::HandleSignal);
signal(SIGINT, server::SignalUtil::HandleSignal);
signal(SIGUSR1, server::SignalUtil::HandleSignal);
signal(SIGSEGV, server::SignalUtil::HandleSignal);
signal(SIGUSR2, server::SignalUtil::HandleSignal);
signal(SIGTERM, server::SignalUtil::HandleSignal);
signal(SIGHUP, zilliz::milvus::server::SignalUtil::HandleSignal);
signal(SIGINT, zilliz::milvus::server::SignalUtil::HandleSignal);
signal(SIGUSR1, zilliz::milvus::server::SignalUtil::HandleSignal);
signal(SIGSEGV, zilliz::milvus::server::SignalUtil::HandleSignal);
signal(SIGUSR2, zilliz::milvus::server::SignalUtil::HandleSignal);
signal(SIGTERM, zilliz::milvus::server::SignalUtil::HandleSignal);
server::Server &server = server::Server::GetInstance();
zilliz::milvus::server::Server &server = zilliz::milvus::server::Server::GetInstance();
server.Init(start_daemonized, pid_filename, config_filename, log_config_file);
server.Start();

2
cpp/src/metrics/PrometheusMetrics.cpp
View File

@ -31,7 +31,7 @@ ErrorCode
PrometheusMetrics::Init() {
try {
Config &config = Config::GetInstance();
Status s = config.GetMetricConfigAutoBootup(startup_);
Status s = config.GetMetricConfigEnableMonitor(startup_);
if (!s.ok()) return s.code();
if (!startup_) return SERVER_SUCCESS;

16
cpp/src/scheduler/Algorithm.cpp
View File

@ -16,20 +16,23 @@
// under the License.
#include "Algorithm.h"
#include "scheduler/Algorithm.h"
#include <limits>
#include <unordered_map>
#include <utility>
namespace zilliz {
namespace milvus {
namespace scheduler {
constexpr uint64_t MAXINT = std::numeric_limits<uint32_t >::max();
constexpr uint64_t MAXINT = std::numeric_limits<uint32_t>::max();
uint64_t
ShortestPath(const ResourcePtr &src,
const ResourcePtr &dest,
const ResourceMgrPtr &res_mgr,
std::vector<std::string> &path) {
std::vector<std::vector<std::string>> paths;
uint64_t num_of_resources = res_mgr->GetAllResources().size();
@ -53,7 +56,6 @@ ShortestPath(const ResourcePtr &src,
std::vector<bool> vis(num_of_resources, false);
std::vector<uint64_t> dis(num_of_resources, MAXINT);
for (auto &res : res_mgr->GetAllResources()) {
auto cur_node = std::static_pointer_cast<Node>(res);
auto cur_neighbours = cur_node->GetNeighbours();
@ -105,6 +107,6 @@ ShortestPath(const ResourcePtr &src,
return dis[name_id_map.at(dest->name())];
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

8
cpp/src/scheduler/Algorithm.h
View File

@ -30,8 +30,8 @@ uint64_t
ShortestPath(const ResourcePtr &src,
const ResourcePtr &dest,
const ResourceMgrPtr &res_mgr,
std::vector<std::string>& path);
std::vector<std::string> &path);
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

7
cpp/src/scheduler/Definition.h
View File

@ -30,7 +30,6 @@
#include "db/engine/EngineFactory.h"
#include "db/engine/ExecutionEngine.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -43,6 +42,6 @@ using EngineFactory = engine::EngineFactory;
using EngineType = engine::EngineType;
using MetricType = engine::MetricType;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

18
cpp/src/scheduler/JobMgr.cpp
View File

@ -15,19 +15,19 @@
// specific language governing permissions and limitations
// under the License.
#include "JobMgr.h"
#include "scheduler/JobMgr.h"
#include "task/Task.h"
#include "TaskCreator.h"
#include <utility>
namespace zilliz {
namespace milvus {
namespace scheduler {
using namespace engine;
JobMgr::JobMgr(ResourceMgrPtr res_mgr)
: res_mgr_(std::move(res_mgr)) {}
: res_mgr_(std::move(res_mgr)) {
}
void
JobMgr::Start() {
@ -59,7 +59,9 @@ void
JobMgr::worker_function() {
while (running_) {
std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [this] { return !queue_.empty(); });
cv_.wait(lock, [this] {
return !queue_.empty();
});
auto job = queue_.front();
queue_.pop();
lock.unlock();
@ -84,6 +86,6 @@ JobMgr::build_task(const JobPtr &job) {
return TaskCreator::Create(job);
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

18
cpp/src/scheduler/JobMgr.h
View File

@ -31,15 +31,13 @@
#include "task/Task.h"
#include "ResourceMgr.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class JobMgr {
public:
explicit
JobMgr(ResourceMgrPtr res_mgr);
public:
explicit JobMgr(ResourceMgrPtr res_mgr);
void
Start();
@ -47,18 +45,18 @@ public:
void
Stop();
public:
public:
void
Put(const JobPtr &job);
private:
private:
void
worker_function();
std::vector<TaskPtr>
build_task(const JobPtr &job);
private:
private:
bool running_ = false;
std::queue<JobPtr> queue_;
@ -72,6 +70,6 @@ private:
using JobMgrPtr = std::shared_ptr<JobMgr>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

9
cpp/src/scheduler/ResourceFactory.cpp
View File

@ -16,8 +16,7 @@
// under the License.
#include "ResourceFactory.h"
#include "scheduler/ResourceFactory.h"
namespace zilliz {
namespace milvus {
@ -40,6 +39,6 @@ ResourceFactory::Create(const std::string &name,
}
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

11
cpp/src/scheduler/ResourceFactory.h
View File

@ -25,13 +25,12 @@
#include "resource/GpuResource.h"
#include "resource/DiskResource.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class ResourceFactory {
public:
public:
static std::shared_ptr<Resource>
Create(const std::string &name,
const std::string &type,
@ -40,8 +39,6 @@ public:
bool enable_executor = true);
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

14
cpp/src/scheduler/ResourceMgr.cpp
View File

@ -16,15 +16,13 @@
// specific language governing permissions and limitations
// under the License.
#include "ResourceMgr.h"
#include "scheduler/ResourceMgr.h"
#include "utils/Log.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
void
ResourceMgr::Start() {
std::lock_guard<std::mutex> lck(resources_mutex_);
@ -186,7 +184,9 @@ void
ResourceMgr::event_process() {
while (running_) {
std::unique_lock<std::mutex> lock(event_mutex_);
event_cv_.wait(lock, [this] { return !queue_.empty(); });
event_cv_.wait(lock, [this] {
return !queue_.empty();
});
auto event = queue_.front();
queue_.pop();
@ -201,6 +201,6 @@ ResourceMgr::event_process() {
}
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

24
cpp/src/scheduler/ResourceMgr.h
View File

@ -22,21 +22,21 @@
#include <memory>
#include <mutex>
#include <queue>
#include <utility>
#include <condition_variable>
#include "resource/Resource.h"
#include "utils/Log.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class ResourceMgr {
public:
public:
ResourceMgr() = default;
public:
public:
/******** Management Interface ********/
void
Start();
@ -58,7 +58,7 @@ public:
subscriber_ = std::move(subscriber);
}
public:
public:
/******** Management Interface ********/
inline std::vector<ResourceWPtr> &
GetDiskResources() {
@ -89,10 +89,10 @@ public:
uint64_t
GetNumGpuResource() const;
public:
public:
// TODO: add stats interface(low)
public:
public:
/******** Utility Functions ********/
std::string
Dump();
@ -100,14 +100,14 @@ public:
std::string
DumpTaskTables();
private:
private:
void
post_event(const EventPtr &event);
void
event_process();
private:
private:
bool running_ = false;
std::vector<ResourceWPtr> disk_resources_;
@ -120,13 +120,11 @@ private:
std::condition_variable event_cv_;
std::thread worker_thread_;
};
using ResourceMgrPtr = std::shared_ptr<ResourceMgr>;
using ResourceMgrWPtr = std::weak_ptr<ResourceMgr>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

13
cpp/src/scheduler/SchedInst.cpp
View File

@ -16,12 +16,16 @@
// under the License.
#include "SchedInst.h"
#include "scheduler/SchedInst.h"
#include "server/Config.h"
#include "ResourceFactory.h"
#include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "Utils.h"
#include <vector>
#include <set>
#include <utility>
#include <string>
namespace zilliz {
namespace milvus {
@ -165,6 +169,7 @@ StopSchedulerService() {
SchedInst::GetInstance()->Stop();
ResMgrInst::GetInstance()->Stop();
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

19
cpp/src/scheduler/SchedInst.h
View File

@ -24,13 +24,12 @@
#include <mutex>
#include <memory>
namespace zilliz {
namespace milvus {
namespace scheduler {
class ResMgrInst {
public:
public:
static ResourceMgrPtr
GetInstance() {
if (instance == nullptr) {
@ -42,13 +41,13 @@ public:
return instance;
}
private:
private:
static ResourceMgrPtr instance;
static std::mutex mutex_;
};
class SchedInst {
public:
public:
static SchedulerPtr
GetInstance() {
if (instance == nullptr) {
@ -60,13 +59,13 @@ public:
return instance;
}
private:
private:
static SchedulerPtr instance;
static std::mutex mutex_;
};
class JobMgrInst {
public:
public:
static scheduler::JobMgrPtr
GetInstance() {
if (instance == nullptr) {
@ -78,7 +77,7 @@ public:
return instance;
}
private:
private:
static scheduler::JobMgrPtr instance;
static std::mutex mutex_;
};
@ -89,6 +88,6 @@ StartSchedulerService();
void
StopSchedulerService();
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

17
cpp/src/scheduler/Scheduler.cpp
View File

@ -15,13 +15,13 @@
// specific language governing permissions and limitations
// under the License.
#include "src/cache/GpuCacheMgr.h"
#include "scheduler/Scheduler.h"
#include "cache/GpuCacheMgr.h"
#include "event/LoadCompletedEvent.h"
#include "Scheduler.h"
#include "action/Action.h"
#include "Algorithm.h"
#include <utility>
namespace zilliz {
namespace milvus {
@ -43,7 +43,6 @@ Scheduler::Scheduler(ResourceMgrWPtr res_mgr)
std::bind(&Scheduler::OnFinishTask, this, std::placeholders::_1)));
}
void
Scheduler::Start() {
running_ = true;
@ -79,7 +78,9 @@ void
Scheduler::worker_function() {
while (running_) {
std::unique_lock<std::mutex> lock(event_mutex_);
event_cv_.wait(lock, [this] { return !event_queue_.empty(); });
event_cv_.wait(lock, [this] {
return !event_queue_.empty();
});
auto event = event_queue_.front();
event_queue_.pop();
if (event == nullptr) {
@ -142,6 +143,6 @@ Scheduler::OnTaskTableUpdated(const EventPtr &event) {
}
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

21
cpp/src/scheduler/Scheduler.h
View File

@ -22,22 +22,20 @@
#include <mutex>
#include <thread>
#include <queue>
#include <unordered_map>
#include "resource/Resource.h"
#include "ResourceMgr.h"
#include "utils/Log.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
// TODO: refactor, not friendly to unittest, logical in framework code
class Scheduler {
public:
explicit
Scheduler(ResourceMgrWPtr res_mgr);
public:
explicit Scheduler(ResourceMgrWPtr res_mgr);
Scheduler(const Scheduler &) = delete;
Scheduler(Scheduler &&) = delete;
@ -66,7 +64,7 @@ public:
std::string
Dump();
private:
private:
/******** Events ********/
/*
@ -106,7 +104,7 @@ private:
void
OnTaskTableUpdated(const EventPtr &event);
private:
private:
/*
* Dispatch event to event handler;
*/
@ -119,7 +117,7 @@ private:
void
worker_function();
private:
private:
bool running_;
std::unordered_map<uint64_t, std::function<void(EventPtr)>> event_register_;
@ -133,7 +131,6 @@ private:
using SchedulerPtr = std::shared_ptr<Scheduler>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

13
cpp/src/scheduler/TaskCreator.cpp
View File

@ -15,11 +15,10 @@
// specific language governing permissions and limitations
// under the License.
#include <src/scheduler/tasklabel/BroadcastLabel.h>
#include "TaskCreator.h"
#include "scheduler/TaskCreator.h"
#include "scheduler/tasklabel/BroadcastLabel.h"
#include "tasklabel/DefaultLabel.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -64,8 +63,6 @@ TaskCreator::Create(const DeleteJobPtr &job) {
return tasks;
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

11
cpp/src/scheduler/TaskCreator.h
View File

@ -34,17 +34,16 @@
#include "task/SearchTask.h"
#include "task/DeleteTask.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class TaskCreator {
public:
public:
static std::vector<TaskPtr>
Create(const JobPtr &job);
public:
public:
static std::vector<TaskPtr>
Create(const SearchJobPtr &job);
@ -52,6 +51,6 @@ public:
Create(const DeleteJobPtr &job);
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

15
cpp/src/scheduler/TaskTable.cpp
View File

@ -16,7 +16,7 @@
// under the License.
#include "TaskTable.h"
#include "scheduler/TaskTable.h"
#include "event/TaskTableUpdatedEvent.h"
#include "Utils.h"
@ -24,7 +24,6 @@
#include <sstream>
#include <ctime>
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -75,6 +74,7 @@ TaskTableItem::Load() {
}
return false;
}
bool
TaskTableItem::Loaded() {
std::unique_lock<std::mutex> lock(mutex);
@ -86,6 +86,7 @@ TaskTableItem::Loaded() {
}
return false;
}
bool
TaskTableItem::Execute() {
std::unique_lock<std::mutex> lock(mutex);
@ -97,6 +98,7 @@ TaskTableItem::Execute() {
}
return false;
}
bool
TaskTableItem::Executed() {
std::unique_lock<std::mutex> lock(mutex);
@ -109,6 +111,7 @@ TaskTableItem::Executed() {
}
return false;
}
bool
TaskTableItem::Move() {
std::unique_lock<std::mutex> lock(mutex);
@ -120,6 +123,7 @@ TaskTableItem::Move() {
}
return false;
}
bool
TaskTableItem::Moved() {
std::unique_lock<std::mutex> lock(mutex);
@ -206,7 +210,6 @@ TaskTable::Put(std::vector<TaskPtr> &tasks) {
}
}
TaskTableItemPtr
TaskTable::Get(uint64_t index) {
return table_[index];
@ -232,6 +235,6 @@ TaskTable::Dump() {
return ss.str();
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

36
cpp/src/scheduler/TaskTable.h
View File

@ -20,11 +20,13 @@
#include <vector>
#include <deque>
#include <mutex>
#include <memory>
#include <utility>
#include <string>
#include "task/SearchTask.h"
#include "event/Event.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -52,7 +54,8 @@ struct TaskTimestamp {
};
struct TaskTableItem {
TaskTableItem() : id(0), task(nullptr), state(TaskTableItemState::INVALID), mutex() {}
TaskTableItem() : id(0), task(nullptr), state(TaskTableItemState::INVALID), mutex() {
}
TaskTableItem(const TaskTableItem &src) = delete;
TaskTableItem(TaskTableItem &&) = delete;
@ -91,7 +94,7 @@ struct TaskTableItem {
using TaskTableItemPtr = std::shared_ptr<TaskTableItem>;
class TaskTable {
public:
public:
TaskTable() = default;
TaskTable(const TaskTable &) = delete;
@ -145,24 +148,28 @@ public:
return table_.size();
}
public:
public:
TaskTableItemPtr &
operator[](uint64_t index) {
return table_[index];
}
std::deque<TaskTableItemPtr>::iterator begin() { return table_.begin(); }
std::deque<TaskTableItemPtr>::iterator end() { return table_.end(); }
std::deque<TaskTableItemPtr>::iterator begin() {
return table_.begin();
}
public:
std::deque<TaskTableItemPtr>::iterator end() {
return table_.end();
}
public:
std::vector<uint64_t>
PickToLoad(uint64_t limit);
std::vector<uint64_t>
PickToExecute(uint64_t limit);
public:
public:
/******** Action ********/
// TODO: bool to Status
@ -227,14 +234,14 @@ public:
return table_[index]->Moved();
}
public:
public:
/*
* Dump;
*/
std::string
Dump();
private:
private:
std::uint64_t id_ = 0;
mutable std::mutex id_mutex_;
std::deque<TaskTableItemPtr> table_;
@ -246,7 +253,6 @@ private:
uint64_t last_finish_ = -1;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

9
cpp/src/scheduler/Utils.cpp
View File

@ -16,12 +16,11 @@
// under the License.
#include "Utils.h"
#include "scheduler/Utils.h"
#include <chrono>
#include <cuda_runtime.h>
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -41,6 +40,6 @@ get_num_gpu() {
return n_devices;
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

7
cpp/src/scheduler/Utils.h
View File

@ -18,7 +18,6 @@
#include <cstdint>
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -29,6 +28,6 @@ get_current_timestamp();
uint64_t
get_num_gpu();
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

15
cpp/src/scheduler/action/Action.h
View File

@ -17,16 +17,17 @@
#pragma once
#include "../resource/Resource.h"
#include "../ResourceMgr.h"
#include "scheduler/resource/Resource.h"
#include "scheduler/ResourceMgr.h"
#include <memory>
namespace zilliz {
namespace milvus {
namespace scheduler {
class Action {
public:
public:
static void
PushTaskToNeighbourRandomly(const TaskPtr &task, const ResourcePtr &self);
@ -43,10 +44,8 @@ public:
SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr,
ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event);
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

22
cpp/src/scheduler/action/PushTaskToNeighbour.cpp
View File

@ -22,7 +22,6 @@
#include "src/cache/GpuCacheMgr.h"
#include "Action.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -57,13 +56,12 @@ get_neighbours_with_connetion(const ResourcePtr &self) {
return neighbours;
}
void
Action::PushTaskToNeighbourRandomly(const TaskPtr &task,
const ResourcePtr &self) {
auto neighbours = get_neighbours_with_connetion(self);
if (not neighbours.empty()) {
std::vector<uint64_t > speeds;
std::vector<uint64_t> speeds;
uint64_t total_speed = 0;
for (auto &neighbour : neighbours) {
uint64_t speed = neighbour.second.speed();
@ -87,7 +85,6 @@ Action::PushTaskToNeighbourRandomly(const TaskPtr &task,
} else {
//TODO: process
}
}
void
@ -99,14 +96,14 @@ Action::PushTaskToAllNeighbour(const TaskPtr &task, const ResourcePtr &self) {
}
void
Action::PushTaskToResource(const TaskPtr& task, const ResourcePtr& dest) {
Action::PushTaskToResource(const TaskPtr &task, const ResourcePtr &dest) {
dest->task_table().Put(task);
}
void
Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr,
ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event) {
ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event) {
if (not resource->HasExecutor() && event->task_table_item_->Move()) {
auto task = event->task_table_item_->task;
auto search_task = std::static_pointer_cast<XSearchTask>(task);
@ -135,8 +132,8 @@ Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr,
void
Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr,
ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event) {
ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event) {
auto task = event->task_table_item_->task;
if (resource->type() == ResourceType::DISK) {
// step 1: calculate shortest path per resource, from disk to compute resource
@ -181,7 +178,6 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr,
}
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

15
cpp/src/scheduler/event/Event.h
View File

@ -18,6 +18,8 @@
#pragma once
#include <memory>
#include <utility>
#include <string>
namespace zilliz {
namespace milvus {
@ -33,11 +35,12 @@ enum class EventType {
class Resource;
class Event {
public:
public:
explicit
Event(EventType type, std::weak_ptr<Resource> resource)
: type_(type),
resource_(std::move(resource)) {}
resource_(std::move(resource)) {
}
inline EventType
Type() const {
@ -49,13 +52,13 @@ public:
friend std::ostream &operator<<(std::ostream &out, const Event &event);
public:
public:
EventType type_;
std::weak_ptr<Resource> resource_;
};
using EventPtr = std::shared_ptr<Event>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

22
cpp/src/scheduler/event/EventDump.cpp
View File

@ -22,36 +22,40 @@
#include "FinishTaskEvent.h"
#include "TaskTableUpdatedEvent.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
std::ostream &operator<<(std::ostream &out, const Event &event) {
std::ostream &
operator<<(std::ostream &out, const Event &event) {
out << event.Dump();
return out;
}
std::ostream &operator<<(std::ostream &out, const StartUpEvent &event) {
std::ostream &
operator<<(std::ostream &out, const StartUpEvent &event) {
out << event.Dump();
return out;
}
std::ostream &operator<<(std::ostream &out, const LoadCompletedEvent &event) {
std::ostream &
operator<<(std::ostream &out, const LoadCompletedEvent &event) {
out << event.Dump();
return out;
}
std::ostream &operator<<(std::ostream &out, const FinishTaskEvent &event) {
std::ostream &
operator<<(std::ostream &out, const FinishTaskEvent &event) {
out << event.Dump();
return out;
}
std::ostream &operator<<(std::ostream &out, const TaskTableUpdatedEvent &event) {
std::ostream &
operator<<(std::ostream &out, const TaskTableUpdatedEvent &event) {
out << event.Dump();
return out;
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

18
cpp/src/scheduler/event/FinishTaskEvent.h
View File

@ -17,18 +17,22 @@
#pragma once
#include "Event.h"
#include "scheduler/event/Event.h"
#include <memory>
#include <utility>
#include <string>
namespace zilliz {
namespace milvus {
namespace scheduler {
class FinishTaskEvent : public Event {
public:
public:
FinishTaskEvent(std::weak_ptr<Resource> resource, TaskTableItemPtr task_table_item)
: Event(EventType::FINISH_TASK, std::move(resource)),
task_table_item_(std::move(task_table_item)) {}
task_table_item_(std::move(task_table_item)) {
}
inline std::string
Dump() const override {
@ -37,10 +41,10 @@ public:
friend std::ostream &operator<<(std::ostream &out, const FinishTaskEvent &event);
public:
public:
TaskTableItemPtr task_table_item_;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

20
cpp/src/scheduler/event/LoadCompletedEvent.h
View File

@ -17,19 +17,23 @@
#pragma once
#include "Event.h"
#include "../TaskTable.h"
#include "scheduler/event/Event.h"
#include "scheduler/TaskTable.h"
#include <memory>
#include <utility>
#include <string>
namespace zilliz {
namespace milvus {
namespace scheduler {
class LoadCompletedEvent : public Event {
public:
public:
LoadCompletedEvent(std::weak_ptr<Resource> resource, TaskTableItemPtr task_table_item)
: Event(EventType::LOAD_COMPLETED, std::move(resource)),
task_table_item_(std::move(task_table_item)) {}
task_table_item_(std::move(task_table_item)) {
}
inline std::string
Dump() const override {
@ -38,10 +42,10 @@ public:
friend std::ostream &operator<<(std::ostream &out, const LoadCompletedEvent &event);
public:
public:
TaskTableItemPtr task_table_item_;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

19
cpp/src/scheduler/event/StartUpEvent.h
View File

@ -17,18 +17,21 @@
#pragma once
#include "Event.h"
#include "scheduler/event/Event.h"
#include <memory>
#include <utility>
#include <string>
namespace zilliz {
namespace milvus {
namespace scheduler {
class StartUpEvent : public Event {
public:
explicit
StartUpEvent(std::weak_ptr<Resource> resource)
: Event(EventType::START_UP, std::move(resource)) {}
public:
explicit StartUpEvent(std::weak_ptr<Resource> resource)
: Event(EventType::START_UP, std::move(resource)) {
}
inline std::string
Dump() const override {
@ -38,6 +41,6 @@ public:
friend std::ostream &operator<<(std::ostream &out, const StartUpEvent &event);
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

18
cpp/src/scheduler/event/TaskTableUpdatedEvent.h
View File

@ -19,16 +19,19 @@
#include "Event.h"
#include <memory>
#include <utility>
#include <string>
namespace zilliz {
namespace milvus {
namespace scheduler {
class TaskTableUpdatedEvent : public Event {
public:
explicit
TaskTableUpdatedEvent(std::weak_ptr<Resource> resource)
: Event(EventType::TASK_TABLE_UPDATED, std::move(resource)) {}
public:
explicit TaskTableUpdatedEvent(std::weak_ptr<Resource> resource)
: Event(EventType::TASK_TABLE_UPDATED, std::move(resource)) {
}
inline std::string
Dump() const override {
@ -38,7 +41,6 @@ public:
friend std::ostream &operator<<(std::ostream &out, const TaskTableUpdatedEvent &event);
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

23
cpp/src/scheduler/job/DeleteJob.cpp
View File

@ -15,8 +15,9 @@
// specific language governing permissions and limitations
// under the License.
#include "DeleteJob.h"
#include "scheduler/job/DeleteJob.h"
#include <utility>
namespace zilliz {
namespace milvus {
@ -29,15 +30,20 @@ DeleteJob::DeleteJob(JobId id,
: Job(id, JobType::DELETE),
table_id_(std::move(table_id)),
meta_ptr_(std::move(meta_ptr)),
num_resource_(num_resource) {}
num_resource_(num_resource) {
}
void DeleteJob::WaitAndDelete() {
void
DeleteJob::WaitAndDelete() {
std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [&] { return done_resource == num_resource_; });
cv_.wait(lock, [&] {
return done_resource == num_resource_;
});
meta_ptr_->DeleteTableFiles(table_id_);
}
void DeleteJob::ResourceDone() {
void
DeleteJob::ResourceDone() {
{
std::lock_guard<std::mutex> lock(mutex_);
++done_resource;
@ -45,7 +51,6 @@ void DeleteJob::ResourceDone() {
cv_.notify_one();
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

16
cpp/src/scheduler/job/DeleteJob.h
View File

@ -30,26 +30,25 @@
#include "Job.h"
#include "db/meta/Meta.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class DeleteJob : public Job {
public:
public:
DeleteJob(JobId id,
std::string table_id,
engine::meta::MetaPtr meta_ptr,
uint64_t num_resource);
public:
public:
void
WaitAndDelete();
void
ResourceDone();
public:
public:
std::string
table_id() const {
return table_id_;
@ -60,7 +59,7 @@ public:
return meta_ptr_;
}
private:
private:
std::string table_id_;
engine::meta::MetaPtr meta_ptr_;
@ -72,7 +71,6 @@ private:
using DeleteJobPtr = std::shared_ptr<DeleteJob>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

17
cpp/src/scheduler/job/Job.h
View File

@ -27,7 +27,6 @@
#include <condition_variable>
#include <memory>
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -42,7 +41,7 @@ enum class JobType {
using JobId = std::uint64_t;
class Job {
public:
public:
inline JobId
id() const {
return id_;
@ -53,10 +52,11 @@ public:
return type_;
}
protected:
Job(JobId id, JobType type) : id_(id), type_(type) {}
protected:
Job(JobId id, JobType type) : id_(id), type_(type) {
}
private:
private:
JobId id_;
JobType type_;
};
@ -64,7 +64,6 @@ private:
using JobPtr = std::shared_ptr<Job>;
using JobWPtr = std::weak_ptr<Job>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

23
cpp/src/scheduler/job/SearchJob.cpp
View File

@ -15,11 +15,9 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/job/SearchJob.h"
#include "utils/Log.h"
#include "SearchJob.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -33,7 +31,8 @@ SearchJob::SearchJob(zilliz::milvus::scheduler::JobId id,
topk_(topk),
nq_(nq),
nprobe_(nprobe),
vectors_(vectors) {}
vectors_(vectors) {
}
bool
SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) {
@ -48,11 +47,12 @@ SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) {
return true;
}
void
SearchJob::WaitResult() {
std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [this] { return index_files_.empty(); });
cv_.wait(lock, [this] {
return index_files_.empty();
});
SERVER_LOG_DEBUG << "SearchJob " << id() << " all done";
}
@ -69,14 +69,11 @@ SearchJob::GetResult() {
return result_;
}
Status&
Status &
SearchJob::GetStatus() {
return status_;
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

19
cpp/src/scheduler/job/SearchJob.h
View File

@ -26,16 +26,15 @@
#include <mutex>
#include <condition_variable>
#include <memory>
#include <utility>
#include "Job.h"
#include "db/meta/MetaTypes.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
using engine::meta::TableFileSchemaPtr;
using Id2IndexMap = std::unordered_map<size_t, TableFileSchemaPtr>;
@ -43,10 +42,10 @@ using Id2DistanceMap = std::vector<std::pair<int64_t, double>>;
using ResultSet = std::vector<Id2DistanceMap>;
class SearchJob : public Job {
public:
public:
SearchJob(JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, const float *vectors);
public:
public:
bool
AddIndexFile(const TableFileSchemaPtr &index_file);
@ -62,7 +61,7 @@ public:
Status &
GetStatus();
public:
public:
uint64_t
topk() const {
return topk_;
@ -77,6 +76,7 @@ public:
nprobe() const {
return nprobe_;
}
const float *
vectors() const {
return vectors_;
@ -87,7 +87,7 @@ public:
return index_files_;
}
private:
private:
uint64_t topk_ = 0;
uint64_t nq_ = 0;
uint64_t nprobe_ = 0;
@ -105,7 +105,6 @@ private:
using SearchJobPtr = std::shared_ptr<SearchJob>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

18
cpp/src/scheduler/resource/Connection.h
View File

@ -19,17 +19,18 @@
#include <string>
#include <sstream>
#include <utility>
namespace zilliz {
namespace milvus {
namespace scheduler {
class Connection {
public:
public:
// TODO: update construct function, speed: double->uint64_t
Connection(std::string name, double speed)
: name_(std::move(name)), speed_(speed) {}
: name_(std::move(name)), speed_(speed) {
}
const std::string &
name() const {
@ -46,7 +47,7 @@ public:
return 1024 / speed_;
}
public:
public:
std::string
Dump() const {
std::stringstream ss;
@ -54,12 +55,11 @@ public:
return ss.str();
}
private:
private:
std::string name_;
uint64_t speed_;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

21
cpp/src/scheduler/resource/CpuResource.cpp
View File

@ -16,29 +16,34 @@
// under the License.
#include "CpuResource.h"
#include "scheduler/resource/CpuResource.h"
#include <utility>
namespace zilliz {
namespace milvus {
namespace scheduler {
std::ostream &operator<<(std::ostream &out, const CpuResource &resource) {
std::ostream &
operator<<(std::ostream &out, const CpuResource &resource) {
out << resource.Dump();
return out;
}
CpuResource::CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor)
: Resource(std::move(name), ResourceType::CPU, device_id, enable_loader, enable_executor) {}
: Resource(std::move(name), ResourceType::CPU, device_id, enable_loader, enable_executor) {
}
void CpuResource::LoadFile(TaskPtr task) {
void
CpuResource::LoadFile(TaskPtr task) {
task->Load(LoadType::DISK2CPU, 0);
}
void CpuResource::Process(TaskPtr task) {
void
CpuResource::Process(TaskPtr task) {
task->Execute();
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

11
cpp/src/scheduler/resource/CpuResource.h
View File

@ -21,13 +21,12 @@
#include "Resource.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class CpuResource : public Resource {
public:
public:
explicit
CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
@ -38,7 +37,7 @@ public:
friend std::ostream &operator<<(std::ostream &out, const CpuResource &resource);
protected:
protected:
void
LoadFile(TaskPtr task) override;
@ -46,6 +45,6 @@ protected:
Process(TaskPtr task) override;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

22
cpp/src/scheduler/resource/DiskResource.cpp
View File

@ -15,14 +15,17 @@
// specific language governing permissions and limitations
// under the License.
#include "DiskResource.h"
#include "scheduler/resource/DiskResource.h"
#include <string>
#include <utility>
namespace zilliz {
namespace milvus {
namespace scheduler {
std::ostream &operator<<(std::ostream &out, const DiskResource &resource) {
std::ostream &
operator<<(std::ostream &out, const DiskResource &resource) {
out << resource.Dump();
return out;
}
@ -31,15 +34,14 @@ DiskResource::DiskResource(std::string name, uint64_t device_id, bool enable_loa
: Resource(std::move(name), ResourceType::DISK, device_id, enable_loader, enable_executor) {
}
void DiskResource::LoadFile(TaskPtr task) {
void
DiskResource::LoadFile(TaskPtr task) {
}
void DiskResource::Process(TaskPtr task) {
void
DiskResource::Process(TaskPtr task) {
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

12
cpp/src/scheduler/resource/DiskResource.h
View File

@ -17,16 +17,16 @@
#pragma once
#include "Resource.h"
#include <string>
namespace zilliz {
namespace milvus {
namespace scheduler {
class DiskResource : public Resource {
public:
public:
explicit
DiskResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
@ -37,7 +37,7 @@ public:
friend std::ostream &operator<<(std::ostream &out, const DiskResource &resource);
protected:
protected:
void
LoadFile(TaskPtr task) override;
@ -45,6 +45,6 @@ protected:
Process(TaskPtr task) override;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

21
cpp/src/scheduler/resource/GpuResource.cpp
View File

@ -16,29 +16,32 @@
// under the License.
#include "GpuResource.h"
#include "scheduler/resource/GpuResource.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
std::ostream &operator<<(std::ostream &out, const GpuResource &resource) {
std::ostream &
operator<<(std::ostream &out, const GpuResource &resource) {
out << resource.Dump();
return out;
}
GpuResource::GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor)
: Resource(std::move(name), ResourceType::GPU, device_id, enable_loader, enable_executor) {}
: Resource(std::move(name), ResourceType::GPU, device_id, enable_loader, enable_executor) {
}
void GpuResource::LoadFile(TaskPtr task) {
void
GpuResource::LoadFile(TaskPtr task) {
task->Load(LoadType::CPU2GPU, device_id_);
}
void GpuResource::Process(TaskPtr task) {
void
GpuResource::Process(TaskPtr task) {
task->Execute();
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

13
cpp/src/scheduler/resource/GpuResource.h
View File

@ -17,16 +17,17 @@
#pragma once
#include "Resource.h"
#include <string>
#include <utility>
namespace zilliz {
namespace milvus {
namespace scheduler {
class GpuResource : public Resource {
public:
public:
explicit
GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
@ -37,7 +38,7 @@ public:
friend std::ostream &operator<<(std::ostream &out, const GpuResource &resource);
protected:
protected:
void
LoadFile(TaskPtr task) override;
@ -45,6 +46,6 @@ protected:
Process(TaskPtr task) override;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

19
cpp/src/scheduler/resource/Node.cpp
View File

@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/resource/Node.h"
#include <atomic>
#include "Node.h"
#include <utility>
namespace zilliz {
namespace milvus {
@ -29,7 +29,8 @@ Node::Node() {
id_ = counter++;
}
std::vector<Neighbour> Node::GetNeighbours() {
std::vector<Neighbour>
Node::GetNeighbours() {
std::lock_guard<std::mutex> lk(mutex_);
std::vector<Neighbour> ret;
for (auto &e : neighbours_) {
@ -38,7 +39,8 @@ std::vector<Neighbour> Node::GetNeighbours() {
return ret;
}
std::string Node::Dump() {
std::string
Node::Dump() {
std::stringstream ss;
ss << "<Node, id=" << std::to_string(id_) << ">::neighbours:" << std::endl;
for (auto &neighbour : neighbours_) {
@ -48,7 +50,8 @@ std::string Node::Dump() {
return ss.str();
}
void Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connection) {
void
Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connection) {
std::lock_guard<std::mutex> lk(mutex_);
if (auto s = neighbour_node.lock()) {
neighbours_.emplace(std::make_pair(s->id_, Neighbour(neighbour_node, connection)));
@ -56,6 +59,6 @@ void Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &conn
// else do nothing, consider it..
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

21
cpp/src/scheduler/resource/Node.h
View File

@ -20,11 +20,11 @@
#include <vector>
#include <memory>
#include <map>
#include <string>
#include "../TaskTable.h"
#include "scheduler/TaskTable.h"
#include "Connection.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -34,8 +34,9 @@ class Node;
using NeighbourNodePtr = std::weak_ptr<Node>;
struct Neighbour {
Neighbour(NeighbourNodePtr nei, Connection conn)
: neighbour_node(nei), connection(conn) {}
Neighbour(NeighbourNodePtr nei, Connection conn)
: neighbour_node(nei), connection(conn) {
}
NeighbourNodePtr neighbour_node;
Connection connection;
@ -43,7 +44,7 @@ struct Neighbour {
// TODO(linxj): return type void -> Status
class Node {
public:
public:
Node();
void
@ -52,11 +53,11 @@ public:
std::vector<Neighbour>
GetNeighbours();
public:
public:
std::string
Dump();
private:
private:
std::mutex mutex_;
uint8_t id_;
std::map<uint8_t, Neighbour> neighbours_;
@ -65,6 +66,6 @@ private:
using NodePtr = std::shared_ptr<Node>;
using NodeWPtr = std::weak_ptr<Node>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

Some files were not shown because too many files have changed in this diff Show More