From 73c9ab43e03b2957faf630ef7ceb8a1fe69b5546 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 22 Oct 2021 14:33:11 +0800 Subject: [PATCH] Simplify cpp (#10413) Signed-off-by: yhmo --- internal/core/src/CMakeLists.txt | 1 - .../core/src/index/knowhere/cache/Cache.h | 104 ------- .../core/src/index/knowhere/cache/Cache.inl | 191 ------------- .../core/src/index/knowhere/cache/CacheMgr.h | 71 ----- .../src/index/knowhere/cache/CacheMgr.inl | 134 --------- .../src/index/knowhere/cache/CpuCacheMgr.cpp | 55 ---- .../src/index/knowhere/cache/CpuCacheMgr.h | 44 --- .../src/index/knowhere/cache/GpuCacheMgr.cpp | 63 ----- .../src/index/knowhere/cache/GpuCacheMgr.h | 51 ---- internal/core/src/index/knowhere/cache/LRU.h | 116 -------- .../src/index/knowhere/log/DeprecatedLog.h | 129 --------- internal/core/src/index/knowhere/log/Log.cpp | 108 -------- internal/core/src/index/knowhere/log/Log.h | 77 ----- .../core/src/index/knowhere/log/LogMgr.cpp | 262 ------------------ internal/core/src/index/knowhere/log/LogMgr.h | 95 ------- internal/core/src/segcore/CMakeLists.txt | 1 - internal/core/src/storage/CMakeLists.txt | 17 -- internal/core/src/storage/StorageCache.cpp | 12 - internal/core/src/storage/StorageCache.h | 32 --- 19 files changed, 1563 deletions(-) delete mode 100644 internal/core/src/index/knowhere/cache/Cache.h delete mode 100644 internal/core/src/index/knowhere/cache/Cache.inl delete mode 100644 internal/core/src/index/knowhere/cache/CacheMgr.h delete mode 100644 internal/core/src/index/knowhere/cache/CacheMgr.inl delete mode 100644 internal/core/src/index/knowhere/cache/CpuCacheMgr.cpp delete mode 100644 internal/core/src/index/knowhere/cache/CpuCacheMgr.h delete mode 100644 internal/core/src/index/knowhere/cache/GpuCacheMgr.cpp delete mode 100644 internal/core/src/index/knowhere/cache/GpuCacheMgr.h delete mode 100644 internal/core/src/index/knowhere/cache/LRU.h delete mode 100644 internal/core/src/index/knowhere/log/DeprecatedLog.h delete mode 100644 internal/core/src/index/knowhere/log/Log.cpp delete mode 100644 internal/core/src/index/knowhere/log/Log.h delete mode 100644 internal/core/src/index/knowhere/log/LogMgr.cpp delete mode 100644 internal/core/src/index/knowhere/log/LogMgr.h delete mode 100644 internal/core/src/storage/CMakeLists.txt delete mode 100644 internal/core/src/storage/StorageCache.cpp delete mode 100644 internal/core/src/storage/StorageCache.h diff --git a/internal/core/src/CMakeLists.txt b/internal/core/src/CMakeLists.txt index 070d8ecd12..a3004a9556 100644 --- a/internal/core/src/CMakeLists.txt +++ b/internal/core/src/CMakeLists.txt @@ -27,7 +27,6 @@ add_subdirectory( exceptions ) add_subdirectory( utils ) add_subdirectory( log ) add_subdirectory( pb ) -add_subdirectory( storage ) add_subdirectory( segcore ) add_subdirectory( query ) add_subdirectory( common ) diff --git a/internal/core/src/index/knowhere/cache/Cache.h b/internal/core/src/index/knowhere/cache/Cache.h deleted file mode 100644 index 34594c4573..0000000000 --- a/internal/core/src/index/knowhere/cache/Cache.h +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include "LRU.h" -#include "utils/Log.h" - -#include -#include -#include -#include - -namespace milvus { -namespace cache { - -template -class Cache { - public: - // mem_capacity, units:GB - Cache(int64_t capacity_gb, int64_t cache_max_count, const std::string& header = ""); - ~Cache() = default; - - int64_t - usage() const { - return usage_; - } - - // unit: BYTE - int64_t - capacity() const { - return capacity_; - } - - // unit: BYTE - void - set_capacity(int64_t capacity); - - double - freemem_percent() const { - return freemem_percent_; - } - - void - set_freemem_percent(double percent) { - freemem_percent_ = percent; - } - - size_t - size() const; - - bool - exists(const std::string& key); - - ItemObj - get(const std::string& key); - - void - insert(const std::string& key, const ItemObj& item); - - void - erase(const std::string& key); - - bool - reserve(const int64_t size); - - void - print(); - - void - clear(); - - private: - void - insert_internal(const std::string& key, const ItemObj& item); - - void - erase_internal(const std::string& key); - - void - free_memory_internal(const int64_t target_size); - - private: - std::string header_; - int64_t usage_; - int64_t capacity_; - double freemem_percent_; - - LRU lru_; - mutable std::mutex mutex_; -}; - -} // namespace cache -} // namespace milvus - -#include "cache/Cache.inl" diff --git a/internal/core/src/index/knowhere/cache/Cache.inl b/internal/core/src/index/knowhere/cache/Cache.inl deleted file mode 100644 index c371efe515..0000000000 --- a/internal/core/src/index/knowhere/cache/Cache.inl +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -namespace milvus { -namespace cache { - -constexpr double DEFAULT_THRESHOLD_PERCENT = 0.7; - -template -Cache::Cache(int64_t capacity, int64_t cache_max_count, const std::string& header) - : header_(header), - usage_(0), - capacity_(capacity), - freemem_percent_(DEFAULT_THRESHOLD_PERCENT), - lru_(cache_max_count) { -} - -template -void -Cache::set_capacity(int64_t capacity) { - std::lock_guard lock(mutex_); - if (capacity > 0) { - capacity_ = capacity; - free_memory_internal(capacity); - } -} - -template -size_t -Cache::size() const { - std::lock_guard lock(mutex_); - return lru_.size(); -} - -template -bool -Cache::exists(const std::string& key) { - std::lock_guard lock(mutex_); - return lru_.exists(key); -} - -template -ItemObj -Cache::get(const std::string& key) { - std::lock_guard lock(mutex_); - if (!lru_.exists(key)) { - return nullptr; - } - return lru_.get(key); -} - -template -void -Cache::insert(const std::string& key, const ItemObj& item) { - std::lock_guard lock(mutex_); - insert_internal(key, item); -} - -template -void -Cache::erase(const std::string& key) { - std::lock_guard lock(mutex_); - erase_internal(key); -} - -template -bool -Cache::reserve(const int64_t item_size) { - std::lock_guard lock(mutex_); - if (item_size > capacity_) { - LOG_SERVER_ERROR_ << header_ << " item size " << (item_size >> 20) << "MB too big to insert into cache capacity" - << (capacity_ >> 20) << "MB"; - return false; - } - if (item_size > capacity_ - usage_) { - free_memory_internal(capacity_ - item_size); - } - return true; -} - -template -void -Cache::clear() { - std::lock_guard lock(mutex_); - lru_.clear(); - usage_ = 0; - LOG_SERVER_DEBUG_ << header_ << " Clear cache !"; -} - - -template -void -Cache::print() { - std::lock_guard lock(mutex_); - size_t cache_count = lru_.size(); - // for (auto it = lru_.begin(); it != lru_.end(); ++it) { - // LOG_SERVER_DEBUG_ << it->first; - // } - LOG_SERVER_DEBUG_ << header_ << " [item count]: " << cache_count << ", [usage] " << (usage_ >> 20) - << "MB, [capacity] " << (capacity_ >> 20) << "MB"; -} - -template -void -Cache::insert_internal(const std::string& key, const ItemObj& item) { - if (item == nullptr) { - return; - } - - size_t item_size = item->Size(); - - // if key already exist, subtract old item size - if (lru_.exists(key)) { - const ItemObj& old_item = lru_.get(key); - usage_ -= old_item->Size(); - } - - // plus new item size - usage_ += item_size; - - // if usage exceed capacity, free some items - if (usage_ > capacity_) { - LOG_SERVER_DEBUG_ << header_ << " Current usage " << (usage_ >> 20) << "MB is too high for capacity " - << (capacity_ >> 20) << "MB, start free memory"; - free_memory_internal(capacity_); - } - - // insert new item - lru_.put(key, item); - LOG_SERVER_DEBUG_ << header_ << " Insert " << key << " size: " << (item_size >> 20) << "MB into cache"; - LOG_SERVER_DEBUG_ << header_ << " Count: " << lru_.size() << ", Usage: " << (usage_ >> 20) << "MB, Capacity: " - << (capacity_ >> 20) << "MB"; -} - -template -void -Cache::erase_internal(const std::string& key) { - if (!lru_.exists(key)) { - return; - } - - const ItemObj& item = lru_.get(key); - size_t item_size = item->Size(); - - lru_.erase(key); - - usage_ -= item_size; - LOG_SERVER_DEBUG_ << header_ << " Erase " << key << " size: " << (item_size >> 20) << "MB from cache"; - LOG_SERVER_DEBUG_ << header_ << " Count: " << lru_.size() << ", Usage: " << (usage_ >> 20) << "MB, Capacity: " - << (capacity_ >> 20) << "MB"; -} - -template -void -Cache::free_memory_internal(const int64_t target_size) { - int64_t threshold = std::min((int64_t)(capacity_ * freemem_percent_), target_size); - int64_t delta_size = usage_ - threshold; - if (delta_size <= 0) { - delta_size = 1; // ensure at least one item erased - } - - std::set key_array; - int64_t released_size = 0; - - auto it = lru_.rbegin(); - while (it != lru_.rend() && released_size < delta_size) { - auto& key = it->first; - auto& obj_ptr = it->second; - - key_array.emplace(key); - released_size += obj_ptr->Size(); - ++it; - } - - LOG_SERVER_DEBUG_ << header_ << " To be released memory size: " << (released_size >> 20) << "MB"; - - for (auto& key : key_array) { - erase_internal(key); - } -} - -} // namespace cache -} // namespace milvus diff --git a/internal/core/src/index/knowhere/cache/CacheMgr.h b/internal/core/src/index/knowhere/cache/CacheMgr.h deleted file mode 100644 index 1d1cfb5ccf..0000000000 --- a/internal/core/src/index/knowhere/cache/CacheMgr.h +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include "Cache.h" -#include "utils/Log.h" - -#include -#include - -namespace milvus { -namespace cache { - -template -class CacheMgr { - public: - virtual uint64_t - ItemCount() const; - - virtual bool - ItemExists(const std::string& key); - - virtual ItemObj - GetItem(const std::string& key); - - virtual void - InsertItem(const std::string& key, const ItemObj& data); - - virtual void - EraseItem(const std::string& key); - - virtual bool - Reserve(const int64_t size); - - virtual void - PrintInfo(); - - virtual void - ClearCache(); - - int64_t - CacheUsage() const; - - int64_t - CacheCapacity() const; - - void - SetCapacity(int64_t capacity); - - protected: - CacheMgr(); - - virtual ~CacheMgr(); - - protected: - std::shared_ptr> cache_; -}; - -} // namespace cache -} // namespace milvus - -#include "cache/CacheMgr.inl" diff --git a/internal/core/src/index/knowhere/cache/CacheMgr.inl b/internal/core/src/index/knowhere/cache/CacheMgr.inl deleted file mode 100644 index ebb44d8f48..0000000000 --- a/internal/core/src/index/knowhere/cache/CacheMgr.inl +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -namespace milvus { -namespace cache { - -template -CacheMgr::CacheMgr() { -} - -template -CacheMgr::~CacheMgr() { -} - -template -uint64_t -CacheMgr::ItemCount() const { - if (cache_ == nullptr) { - LOG_SERVER_ERROR_ << "Cache doesn't exist"; - return 0; - } - return (uint64_t)(cache_->size()); -} - -template -bool -CacheMgr::ItemExists(const std::string& key) { - if (cache_ == nullptr) { - LOG_SERVER_ERROR_ << "Cache doesn't exist"; - return false; - } - return cache_->exists(key); -} - -template -ItemObj -CacheMgr::GetItem(const std::string& key) { - if (cache_ == nullptr) { - LOG_SERVER_ERROR_ << "Cache doesn't exist"; - return nullptr; - } - return cache_->get(key); -} - -template -void -CacheMgr::InsertItem(const std::string& key, const ItemObj& data) { - if (cache_ == nullptr) { - LOG_SERVER_ERROR_ << "Cache doesn't exist"; - return; - } - cache_->insert(key, data); -} - -template -void -CacheMgr::EraseItem(const std::string& key) { - if (cache_ == nullptr) { - LOG_SERVER_ERROR_ << "Cache doesn't exist"; - return; - } - cache_->erase(key); -} - -template -bool -CacheMgr::Reserve(const int64_t size) { - if (cache_ == nullptr) { - LOG_SERVER_ERROR_ << "Cache doesn't exist"; - return false; - } - return cache_->reserve(size); -} - -template -void -CacheMgr::PrintInfo() { - if (cache_ == nullptr) { - LOG_SERVER_ERROR_ << "Cache doesn't exist"; - return; - } - cache_->print(); -} - -template -void -CacheMgr::ClearCache() { - if (cache_ == nullptr) { - LOG_SERVER_ERROR_ << "Cache doesn't exist"; - return; - } - cache_->clear(); -} - -template -int64_t -CacheMgr::CacheUsage() const { - if (cache_ == nullptr) { - LOG_SERVER_ERROR_ << "Cache doesn't exist"; - return 0; - } - return cache_->usage(); -} - -template -int64_t -CacheMgr::CacheCapacity() const { - if (cache_ == nullptr) { - LOG_SERVER_ERROR_ << "Cache doesn't exist"; - return 0; - } - return cache_->capacity(); -} - -template -void -CacheMgr::SetCapacity(int64_t capacity) { - if (cache_ == nullptr) { - LOG_SERVER_ERROR_ << "Cache doesn't exist"; - return; - } - cache_->set_capacity(capacity); -} - -} // namespace cache -} // namespace milvus diff --git a/internal/core/src/index/knowhere/cache/CpuCacheMgr.cpp b/internal/core/src/index/knowhere/cache/CpuCacheMgr.cpp deleted file mode 100644 index fc819f5bb7..0000000000 --- a/internal/core/src/index/knowhere/cache/CpuCacheMgr.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "cache/CpuCacheMgr.h" - -#include - -#include - -#include "utils/Log.h" -#include "value/config/ServerConfig.h" - -namespace milvus { -namespace cache { - -CpuCacheMgr& -CpuCacheMgr::GetInstance() { - static CpuCacheMgr s_mgr; - return s_mgr; -} - -CpuCacheMgr::CpuCacheMgr() { - cache_ = std::make_shared>(config.cache.cache_size(), 1UL << 32, "[CACHE CPU]"); - - if (config.cache.cpu_cache_threshold() > 0.0) { - cache_->set_freemem_percent(config.cache.cpu_cache_threshold()); - } - ConfigMgr::GetInstance().Attach("cache.cache_size", this); -} - -CpuCacheMgr::~CpuCacheMgr() { - ConfigMgr::GetInstance().Detach("cache.cache_size", this); -} - -DataObjPtr -CpuCacheMgr::GetItem(const std::string& key) { - auto ret = CacheMgr::GetItem(key); - return ret; -} - -void -CpuCacheMgr::ConfigUpdate(const std::string& name) { - SetCapacity(config.cache.cache_size()); -} - -} // namespace cache -} // namespace milvus diff --git a/internal/core/src/index/knowhere/cache/CpuCacheMgr.h b/internal/core/src/index/knowhere/cache/CpuCacheMgr.h deleted file mode 100644 index bd53354a28..0000000000 --- a/internal/core/src/index/knowhere/cache/CpuCacheMgr.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include - -#include "cache/CacheMgr.h" -#include "cache/DataObj.h" -#include "value/config/ConfigMgr.h" - -namespace milvus { -namespace cache { - -class CpuCacheMgr : public CacheMgr, public ConfigObserver { - public: - static CpuCacheMgr& - GetInstance(); - - private: - CpuCacheMgr(); - - ~CpuCacheMgr(); - - public: - DataObjPtr - GetItem(const std::string& key) override; - - public: - void - ConfigUpdate(const std::string& name) override; -}; - -} // namespace cache -} // namespace milvus diff --git a/internal/core/src/index/knowhere/cache/GpuCacheMgr.cpp b/internal/core/src/index/knowhere/cache/GpuCacheMgr.cpp deleted file mode 100644 index 9e5be8babe..0000000000 --- a/internal/core/src/index/knowhere/cache/GpuCacheMgr.cpp +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "cache/GpuCacheMgr.h" -#include "utils/Log.h" -#include "value/config/ServerConfig.h" - -#include -#include -#include - -namespace milvus { -namespace cache { - -#ifdef MILVUS_GPU_VERSION -std::mutex GpuCacheMgr::global_mutex_; -std::unordered_map GpuCacheMgr::instance_; - -GpuCacheMgr::GpuCacheMgr(int64_t gpu_id) : gpu_id_(gpu_id) { - std::string header = "[CACHE GPU" + std::to_string(gpu_id) + "]"; - cache_ = std::make_shared>(config.gpu.cache_size(), 1UL << 32, header); - - if (config.gpu.cache_threshold() > 0.0) { - cache_->set_freemem_percent(config.gpu.cache_threshold()); - } - ConfigMgr::GetInstance().Attach("gpu.cache_threshold", this); -} - -GpuCacheMgr::~GpuCacheMgr() { - ConfigMgr::GetInstance().Detach("gpu.cache_threshold", this); -} - -GpuCacheMgrPtr -GpuCacheMgr::GetInstance(int64_t gpu_id) { - if (instance_.find(gpu_id) == instance_.end()) { - std::lock_guard lock(global_mutex_); - if (instance_.find(gpu_id) == instance_.end()) { - instance_[gpu_id] = std::make_shared(gpu_id); - } - } - return instance_[gpu_id]; -} - -void -GpuCacheMgr::ConfigUpdate(const std::string& name) { - std::lock_guard lock(global_mutex_); - for (auto& it : instance_) { - it.second->SetCapacity(config.gpu.cache_size()); - } -} - -#endif - -} // namespace cache -} // namespace milvus diff --git a/internal/core/src/index/knowhere/cache/GpuCacheMgr.h b/internal/core/src/index/knowhere/cache/GpuCacheMgr.h deleted file mode 100644 index a4d2ae9ed0..0000000000 --- a/internal/core/src/index/knowhere/cache/GpuCacheMgr.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include -#include -#include -#include -#include - -#include "cache/CacheMgr.h" -#include "cache/DataObj.h" -#include "value/config/ConfigMgr.h" - -namespace milvus { -namespace cache { - -#ifdef MILVUS_GPU_VERSION -class GpuCacheMgr; -using GpuCacheMgrPtr = std::shared_ptr; -using MutexPtr = std::shared_ptr; - -class GpuCacheMgr : public CacheMgr, public ConfigObserver { - public: - explicit GpuCacheMgr(int64_t gpu_id); - - ~GpuCacheMgr(); - - static GpuCacheMgrPtr - GetInstance(int64_t gpu_id); - - public: - void - ConfigUpdate(const std::string& name) override; - - private: - int64_t gpu_id_; - static std::mutex global_mutex_; - static std::unordered_map instance_; -}; -#endif - -} // namespace cache -} // namespace milvus diff --git a/internal/core/src/index/knowhere/cache/LRU.h b/internal/core/src/index/knowhere/cache/LRU.h deleted file mode 100644 index e8a853cea0..0000000000 --- a/internal/core/src/index/knowhere/cache/LRU.h +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include - -namespace milvus { -namespace cache { - -template -class LRU { - public: - typedef typename std::pair key_value_pair_t; - typedef typename std::list::iterator list_iterator_t; - typedef typename std::list::reverse_iterator reverse_list_iterator_t; - - explicit LRU(size_t max_size) : max_size_(max_size) { - } - - void - put(const key_t& key, const value_t& value) { - auto it = cache_items_map_.find(key); - cache_items_list_.push_front(key_value_pair_t(key, value)); - if (it != cache_items_map_.end()) { - cache_items_list_.erase(it->second); - cache_items_map_.erase(it); - } - cache_items_map_[key] = cache_items_list_.begin(); - - if (cache_items_map_.size() > max_size_) { - auto last = cache_items_list_.end(); - last--; - cache_items_map_.erase(last->first); - cache_items_list_.pop_back(); - } - } - - const value_t& - get(const key_t& key) { - auto it = cache_items_map_.find(key); - if (it == cache_items_map_.end()) { - throw std::range_error("There is no such key in cache"); - } else { - cache_items_list_.splice(cache_items_list_.begin(), cache_items_list_, it->second); - return it->second->second; - } - } - - void - erase(const key_t& key) { - auto it = cache_items_map_.find(key); - if (it != cache_items_map_.end()) { - cache_items_list_.erase(it->second); - cache_items_map_.erase(it); - } - } - - bool - exists(const key_t& key) const { - return cache_items_map_.find(key) != cache_items_map_.end(); - } - - size_t - size() const { - return cache_items_map_.size(); - } - - list_iterator_t - begin() { - iter_ = cache_items_list_.begin(); - return iter_; - } - - list_iterator_t - end() { - return cache_items_list_.end(); - } - - reverse_list_iterator_t - rbegin() { - return cache_items_list_.rbegin(); - } - - reverse_list_iterator_t - rend() { - return cache_items_list_.rend(); - } - - void - clear() { - cache_items_list_.clear(); - cache_items_map_.clear(); - } - - private: - std::list cache_items_list_; - std::unordered_map cache_items_map_; - size_t max_size_; - list_iterator_t iter_; -}; - -} // namespace cache -} // namespace milvus diff --git a/internal/core/src/index/knowhere/log/DeprecatedLog.h b/internal/core/src/index/knowhere/log/DeprecatedLog.h deleted file mode 100644 index 3bf0c1ee87..0000000000 --- a/internal/core/src/index/knowhere/log/DeprecatedLog.h +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -/* - * Please use LOG_MODULE_LEVEL_C macro in member function of class - * and LOG_MODULE_LEVEL_ macro in other functions. - */ - -///////////////////////////////////////////////////////////////////////////////////////////////// -#define SERVER_MODULE_NAME "SERVER" -#define SERVER_MODULE_CLASS_FUNCTION \ - LogOut("[%s][%s::%s][%s] ", SERVER_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, \ - milvus::knowhere::GetThreadName().c_str()) -#define SERVER_MODULE_FUNCTION \ - LogOut("[%s][%s][%s] ", SERVER_MODULE_NAME, __FUNCTION__, milvus::knowhere::GetThreadName().c_str()) - -#define LOG_SERVER_TRACE_C LOG(TRACE) << SERVER_MODULE_CLASS_FUNCTION -#define LOG_SERVER_DEBUG_C LOG(DEBUG) << SERVER_MODULE_CLASS_FUNCTION -#define LOG_SERVER_INFO_C LOG(INFO) << SERVER_MODULE_CLASS_FUNCTION -#define LOG_SERVER_WARNING_C LOG(WARNING) << SERVER_MODULE_CLASS_FUNCTION -#define LOG_SERVER_ERROR_C LOG(ERROR) << SERVER_MODULE_CLASS_FUNCTION -#define LOG_SERVER_FATAL_C LOG(FATAL) << SERVER_MODULE_CLASS_FUNCTION - -#define LOG_SERVER_TRACE_ LOG(TRACE) << SERVER_MODULE_FUNCTION -#define LOG_SERVER_DEBUG_ LOG(DEBUG) << SERVER_MODULE_FUNCTION -#define LOG_SERVER_INFO_ LOG(INFO) << SERVER_MODULE_FUNCTION -#define LOG_SERVER_WARNING_ LOG(WARNING) << SERVER_MODULE_FUNCTION -#define LOG_SERVER_ERROR_ LOG(ERROR) << SERVER_MODULE_FUNCTION -#define LOG_SERVER_FATAL_ LOG(FATAL) << SERVER_MODULE_FUNCTION - -///////////////////////////////////////////////////////////////////////////////////////////////// -#define ENGINE_MODULE_NAME "ENGINE" -#define ENGINE_MODULE_CLASS_FUNCTION \ - LogOut("[%s][%s::%s][%s] ", ENGINE_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, \ - milvus::knowhere::GetThreadName().c_str()) -#define ENGINE_MODULE_FUNCTION \ - LogOut("[%s][%s][%s] ", ENGINE_MODULE_NAME, __FUNCTION__, milvus::knowhere::GetThreadName().c_str()) - -#define LOG_ENGINE_TRACE_C LOG(TRACE) << ENGINE_MODULE_CLASS_FUNCTION -#define LOG_ENGINE_DEBUG_C LOG(DEBUG) << ENGINE_MODULE_CLASS_FUNCTION -#define LOG_ENGINE_INFO_C LOG(INFO) << ENGINE_MODULE_CLASS_FUNCTION -#define LOG_ENGINE_WARNING_C LOG(WARNING) << ENGINE_MODULE_CLASS_FUNCTION -#define LOG_ENGINE_ERROR_C LOG(ERROR) << ENGINE_MODULE_CLASS_FUNCTION -#define LOG_ENGINE_FATAL_C LOG(FATAL) << ENGINE_MODULE_CLASS_FUNCTION - -#define LOG_ENGINE_TRACE_ LOG(TRACE) << ENGINE_MODULE_FUNCTION -#define LOG_ENGINE_DEBUG_ LOG(DEBUG) << ENGINE_MODULE_FUNCTION -#define LOG_ENGINE_INFO_ LOG(INFO) << ENGINE_MODULE_FUNCTION -#define LOG_ENGINE_WARNING_ LOG(WARNING) << ENGINE_MODULE_FUNCTION -#define LOG_ENGINE_ERROR_ LOG(ERROR) << ENGINE_MODULE_FUNCTION -#define LOG_ENGINE_FATAL_ LOG(FATAL) << ENGINE_MODULE_FUNCTION - -///////////////////////////////////////////////////////////////////////////////////////////////// -#define WRAPPER_MODULE_NAME "WRAPPER" -#define WRAPPER_MODULE_CLASS_FUNCTION \ - LogOut("[%s][%s::%s][%s] ", WRAPPER_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, \ - milvus::knowhere::GetThreadName().c_str()) -#define WRAPPER_MODULE_FUNCTION \ - LogOut("[%s][%s][%s] ", WRAPPER_MODULE_NAME, __FUNCTION__, milvus::knowhere::GetThreadName().c_str()) - -#define LOG_WRAPPER_TRACE_C LOG(TRACE) << WRAPPER_MODULE_CLASS_FUNCTION -#define LOG_WRAPPER_DEBUG_C LOG(DEBUG) << WRAPPER_MODULE_CLASS_FUNCTION -#define LOG_WRAPPER_INFO_C LOG(INFO) << WRAPPER_MODULE_CLASS_FUNCTION -#define LOG_WRAPPER_WARNING_C LOG(WARNING) << WRAPPER_MODULE_CLASS_FUNCTION -#define LOG_WRAPPER_ERROR_C LOG(ERROR) << WRAPPER_MODULE_CLASS_FUNCTION -#define LOG_WRAPPER_FATAL_C LOG(FATAL) << WRAPPER_MODULE_CLASS_FUNCTION - -#define LOG_WRAPPER_TRACE_ LOG(TRACE) << WRAPPER_MODULE_FUNCTION -#define LOG_WRAPPER_DEBUG_ LOG(DEBUG) << WRAPPER_MODULE_FUNCTION -#define LOG_WRAPPER_INFO_ LOG(INFO) << WRAPPER_MODULE_FUNCTION -#define LOG_WRAPPER_WARNING_ LOG(WARNING) << WRAPPER_MODULE_FUNCTION -#define LOG_WRAPPER_ERROR_ LOG(ERROR) << WRAPPER_MODULE_FUNCTION -#define LOG_WRAPPER_FATAL_ LOG(FATAL) << WRAPPER_MODULE_FUNCTION - -///////////////////////////////////////////////////////////////////////////////////////////////// -#define STORAGE_MODULE_NAME "STORAGE" -#define STORAGE_MODULE_CLASS_FUNCTION \ - LogOut("[%s][%s::%s][%s] ", STORAGE_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, \ - milvus::knowhere::GetThreadName().c_str()) -#define STORAGE_MODULE_FUNCTION \ - LogOut("[%s][%s][%s] ", STORAGE_MODULE_NAME, __FUNCTION__, milvus::knowhere::GetThreadName().c_str()) - -#define LOG_STORAGE_TRACE_C LOG(TRACE) << STORAGE_MODULE_CLASS_FUNCTION -#define LOG_STORAGE_DEBUG_C LOG(DEBUG) << STORAGE_MODULE_CLASS_FUNCTION -#define LOG_STORAGE_INFO_C LOG(INFO) << STORAGE_MODULE_CLASS_FUNCTION -#define LOG_STORAGE_WARNING_C LOG(WARNING) << STORAGE_MODULE_CLASS_FUNCTION -#define LOG_STORAGE_ERROR_C LOG(ERROR) << STORAGE_MODULE_CLASS_FUNCTION -#define LOG_STORAGE_FATAL_C LOG(FATAL) << STORAGE_MODULE_CLASS_FUNCTION - -#define LOG_STORAGE_TRACE_ LOG(TRACE) << STORAGE_MODULE_FUNCTION -#define LOG_STORAGE_DEBUG_ LOG(DEBUG) << STORAGE_MODULE_FUNCTION -#define LOG_STORAGE_INFO_ LOG(INFO) << STORAGE_MODULE_FUNCTION -#define LOG_STORAGE_WARNING_ LOG(WARNING) << STORAGE_MODULE_FUNCTION -#define LOG_STORAGE_ERROR_ LOG(ERROR) << STORAGE_MODULE_FUNCTION -#define LOG_STORAGE_FATAL_ LOG(FATAL) << STORAGE_MODULE_FUNCTION - -///////////////////////////////////////////////////////////////////////////////////////////////// -#define WAL_MODULE_NAME "WAL" -#define WAL_MODULE_CLASS_FUNCTION \ - LogOut("[%s][%s::%s][%s] ", WAL_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, \ - milvus::knowhere::GetThreadName().c_str()) -#define WAL_MODULE_FUNCTION \ - LogOut("[%s][%s][%s] ", WAL_MODULE_NAME, __FUNCTION__, milvus::knowhere::GetThreadName().c_str()) - -#define LOG_WAL_TRACE_C LOG(TRACE) << WAL_MODULE_CLASS_FUNCTION -#define LOG_WAL_DEBUG_C LOG(DEBUG) << WAL_MODULE_CLASS_FUNCTION -#define LOG_WAL_INFO_C LOG(INFO) << WAL_MODULE_CLASS_FUNCTION -#define LOG_WAL_WARNING_C LOG(WARNING) << WAL_MODULE_CLASS_FUNCTION -#define LOG_WAL_ERROR_C LOG(ERROR) << WAL_MODULE_CLASS_FUNCTION -#define LOG_WAL_FATAL_C LOG(FATAL) << WAL_MODULE_CLASS_FUNCTION - -#define LOG_WAL_TRACE_ LOG(TRACE) << WAL_MODULE_FUNCTION -#define LOG_WAL_DEBUG_ LOG(DEBUG) << WAL_MODULE_FUNCTION -#define LOG_WAL_INFO_ LOG(INFO) << WAL_MODULE_FUNCTION -#define LOG_WAL_WARNING_ LOG(WARNING) << WAL_MODULE_FUNCTION -#define LOG_WAL_ERROR_ LOG(ERROR) << WAL_MODULE_FUNCTION -#define LOG_WAL_FATAL_ LOG(FATAL) << WAL_MODULE_FUNCTION - -///////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/internal/core/src/index/knowhere/log/Log.cpp b/internal/core/src/index/knowhere/log/Log.cpp deleted file mode 100644 index 61b949e1ed..0000000000 --- a/internal/core/src/index/knowhere/log/Log.cpp +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "log/Log.h" -INITIALIZE_EASYLOGGINGPP - -#include -#include -#include -#include -#include -#include - -namespace milvus { - -std::string -LogOut(const char* pattern, ...) { - size_t len = strnlen(pattern, 1024) + 256; - auto str_p = std::make_unique(len); - memset(str_p.get(), 0, len); - - va_list vl; - va_start(vl, pattern); - vsnprintf(str_p.get(), len, pattern, vl); // NOLINT - va_end(vl); - - return std::string(str_p.get()); -} - -void -SetThreadName(const std::string& name) { - // Note: the name cannot exceed 16 bytes - pthread_setname_np(pthread_self(), name.c_str()); -} - -std::string -GetThreadName() { - std::string thread_name = "unamed"; - char name[16]; - size_t len = 16; - auto err = pthread_getname_np(pthread_self(), name, len); - if (not err) { - thread_name = name; - } - - return thread_name; -} - -int64_t -get_now_timestamp() { - auto now = std::chrono::system_clock::now().time_since_epoch(); - return std::chrono::duration_cast(now).count(); -} - -int64_t -get_system_boottime() { - FILE* uptime = fopen("/proc/uptime", "r"); - float since_sys_boot, _; - auto ret = fscanf(uptime, "%f %f", &since_sys_boot, &_); - fclose(uptime); - if (ret != 2) { - throw std::runtime_error("read /proc/uptime failed."); - } - return static_cast(since_sys_boot); -} - -int64_t -get_thread_starttime() { - int64_t tid = gettid(); - int64_t pid = getpid(); - char filename[256]; - snprintf(filename, sizeof(filename), "/proc/%ld/task/%ld/stat", pid, tid); - - int64_t val = 0; - char comm[16], state; - FILE* thread_stat = fopen(filename, "r"); - auto ret = fscanf(thread_stat, "%ld %s %s ", &val, comm, &state); - for (auto i = 4; i < 23; i++) { - ret = fscanf(thread_stat, "%ld ", &val); - if (i == 22) { - break; - } - } - fclose(thread_stat); - if (ret != 1) { - throw std::runtime_error("read " + std::string(filename) + " failed."); - } - return val / sysconf(_SC_CLK_TCK); -} - -int64_t -get_thread_start_timestamp() { - try { - return get_now_timestamp() - get_system_boottime() + get_thread_starttime(); - } catch (...) { - return 0; - } -} - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/log/Log.h b/internal/core/src/index/knowhere/log/Log.h deleted file mode 100644 index dcc5feb697..0000000000 --- a/internal/core/src/index/knowhere/log/Log.h +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include - -#include -#include -#include "easyloggingpp/easylogging++.h" - -#include "log/DeprecatedLog.h" - -namespace milvus { - -#if __GLIBC__ == 2 && __GLIBC_MINOR__ < 30 -#include -#define gettid() syscall(SYS_gettid) -#endif - -// Log message format: %timestamp | %request_id | %level | %collection_name | %client_id | %client_tag | %client_ipport -// | %thread_id | %thread_start_timestamp | %command_tag | %module | %error_code | %message - -#define VAR_REQUEST_ID (context->request_id()) -#define VAR_COLLECTION_NAME (context->collection_name()) -#define VAR_CLIENT_ID ("") -#define VAR_CLIENT_TAG (context->client_tag()) -#define VAR_CLIENT_IPPORT (context->client_ipport()) -#define VAR_THREAD_ID (gettid()) -#define VAR_THREAD_START_TIMESTAMP (get_thread_start_timestamp()) -#define VAR_COMMAND_TAG (context->command_tag()) - -// Use this macro whenever possible -// Depends variables: context Context -#define MLOG(level, module, error_code) \ - LOG(level) << " | " << VAR_REQUEST_ID << " | " << #level << " | " << VAR_COLLECTION_NAME << " | " << VAR_CLIENT_ID \ - << " | " << VAR_CLIENT_TAG << " | " << VAR_CLIENT_IPPORT << " | " << VAR_THREAD_ID << " | " \ - << VAR_THREAD_START_TIMESTAMP << " | " << VAR_COMMAND_TAG << " | " << #module << " | " << error_code \ - << " | " - -// Use in some background process only -#define MLOG_(level, module, error_code) \ - LOG(level) << " | " \ - << "" \ - << " | " << #level << " | " \ - << "" \ - << " | " \ - << "" \ - << " | " \ - << "" \ - << " | " \ - << "" \ - << " | " << VAR_THREAD_ID << " | " << VAR_THREAD_START_TIMESTAMP << " | " \ - << "" \ - << " | " << #module << " | " << error_code << " | " - -std::string -LogOut(const char* pattern, ...); - -void -SetThreadName(const std::string& name); - -std::string -GetThreadName(); - -int64_t -get_thread_start_timestamp(); - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/log/LogMgr.cpp b/internal/core/src/index/knowhere/log/LogMgr.cpp deleted file mode 100644 index 948817a080..0000000000 --- a/internal/core/src/index/knowhere/log/LogMgr.cpp +++ /dev/null @@ -1,262 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "log/LogMgr.h" -#include "utils/Status.h" -#include "value/config/ServerConfig.h" - -namespace milvus { - -int LogMgr::trace_idx = 0; -int LogMgr::global_idx = 0; -int LogMgr::debug_idx = 0; -int LogMgr::info_idx = 0; -int LogMgr::warning_idx = 0; -int LogMgr::error_idx = 0; -int LogMgr::fatal_idx = 0; -int64_t LogMgr::logs_delete_exceeds = 1; -bool LogMgr::enable_log_delete = false; - -Status -LogMgr::InitLog(bool trace_enable, - const std::string& level, - const std::string& logs_path, - const std::string& filename, - int64_t max_log_file_size, - int64_t log_rotate_num, - bool log_to_stdout, - bool log_to_file) { - try { - auto enables = parse_level(level); - enables["trace"] = trace_enable; - LogMgr log_mgr(logs_path); - log_mgr.Default() - .Filename(filename) - .Level(enables) - .To(log_to_stdout, log_to_file) - .Rotate(max_log_file_size, log_rotate_num) - .Setup(); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -// TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename -void -LogMgr::RolloutHandler(const char* filename, std::size_t size, el::Level level) { - char* dirc = strdup(filename); - char* basec = strdup(filename); - char* dir = dirname(dirc); - char* base = basename(basec); - - std::string s(base); - std::vector list = {"\\", " ", "\'", "\"", "*", "\?", "{", "}", ";", "<", - ">", "|", "^", "&", "$", "#", "!", "`", "~"}; - std::string::size_type position; - for (auto& substr : list) { - position = 0; - while ((position = s.find_first_of(substr, position)) != std::string::npos) { - s.insert(position, "\\"); - position += 2; - } - } - std::string m(std::string(dir) + "/" + s); - try { - switch (level) { - case el::Level::Trace: { - rename_and_delete(m, ++trace_idx); - break; - } - case el::Level::Global: { - rename_and_delete(m, ++global_idx); - break; - } - case el::Level::Debug: { - rename_and_delete(m, ++debug_idx); - break; - } - case el::Level::Info: { - rename_and_delete(m, ++info_idx); - break; - } - case el::Level::Warning: { - rename_and_delete(m, ++warning_idx); - break; - } - case el::Level::Error: { - rename_and_delete(m, ++error_idx); - break; - } - case el::Level::Fatal: { - rename_and_delete(m, ++fatal_idx); - break; - } - default: { - break; - } - } - } catch (const std::exception& exc) { - std::cerr << exc.what() << ". Exception throws from RolloutHandler." << std::endl; - } -} - -LogMgr::LogMgr(std::string log_path) : logs_path_(std::move(log_path)) { -} - -LogMgr& -LogMgr::Default() { - el_config_.setToDefault(); - el_config_.setGlobally(el::ConfigurationType::Format, "[%datetime][%level]%msg"); - el_config_.setGlobally(el::ConfigurationType::SubsecondPrecision, "3"); - el_config_.setGlobally(el::ConfigurationType::PerformanceTracking, "false"); - - return *this; -} - -LogMgr& -LogMgr::Filename(const std::string& filename) { - std::string logs_reg_path = logs_path_.rfind('/') == logs_path_.length() - 1 ? logs_path_ : logs_path_ + "/"; - - std::string log_file = logs_reg_path + filename; - - /* Set set log file at Global level to make all level log output to the same log file*/ - el_config_.set(el::Level::Global, el::ConfigurationType::Filename, log_file.c_str()); - - return *this; -} - -LogMgr& -LogMgr::Level(std::unordered_map& enables) { - fiu_do_on("LogMgr.Level.trace_enable_to_false", enables["trace"] = false); - enable(el_config_, el::Level::Trace, enables["trace"]); - - fiu_do_on("LogMgr.Level.info_enable_to_false", enables["info"] = false); - enable(el_config_, el::Level::Info, enables["info"]); - - fiu_do_on("LogMgr.Level.debug_enable_to_false", enables["debug"] = false); - enable(el_config_, el::Level::Debug, enables["debug"]); - - fiu_do_on("LogMgr.Level.warning_enable_to_false", enables["warning"] = false); - enable(el_config_, el::Level::Warning, enables["warning"]); - - fiu_do_on("LogMgr.Level.error_enable_to_false", enables["error"] = false); - enable(el_config_, el::Level::Error, enables["error"]); - - fiu_do_on("LogMgr.Level.fatal_enable_to_false", enables["fatal"] = false); - enable(el_config_, el::Level::Fatal, enables["fatal"]); - - return *this; -} - -LogMgr& -LogMgr::To(bool log_to_stdout, bool log_to_file) { - el_config_.setGlobally(el::ConfigurationType::ToStandardOutput, (log_to_stdout ? "true" : "false")); - el_config_.setGlobally(el::ConfigurationType::ToFile, (log_to_file ? "true" : "false")); - - return *this; -} - -LogMgr& -LogMgr::Rotate(int64_t max_log_file_size, int64_t log_rotate_num) { - fiu_do_on("LogMgr.Rotate.set_max_log_size_small_than_min", max_log_file_size = MAX_LOG_FILE_SIZE_MIN - 1); - if (max_log_file_size < MAX_LOG_FILE_SIZE_MIN || max_log_file_size > MAX_LOG_FILE_SIZE_MAX) { - std::string msg = "max_log_file_size must in range[" + std::to_string(MAX_LOG_FILE_SIZE_MIN) + ", " + - std::to_string(MAX_LOG_FILE_SIZE_MAX) + "], now is " + std::to_string(max_log_file_size); - throw std::runtime_error(msg); - } - - el_config_.setGlobally(el::ConfigurationType::MaxLogFileSize, std::to_string(max_log_file_size)); - el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck); - el::Helpers::installPreRollOutCallback(LogMgr::RolloutHandler); - el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog); - - // set delete_exceeds = 0 means disable throw away log file even they reach certain limit. - if (log_rotate_num != 0) { - fiu_do_on("LogMgr.Rotate.delete_exceeds_small_than_min", log_rotate_num = LOG_ROTATE_NUM_MIN - 1); - if (log_rotate_num < LOG_ROTATE_NUM_MIN || log_rotate_num > LOG_ROTATE_NUM_MAX) { - std::string msg = "log_rotate_num must in range[" + std::to_string(LOG_ROTATE_NUM_MIN) + ", " + - std::to_string(LOG_ROTATE_NUM_MAX) + "], now is " + std::to_string(log_rotate_num); - throw std::runtime_error(msg); - } - - /* global variable */ - enable_log_delete = true; - logs_delete_exceeds = log_rotate_num; - } - - return *this; -} - -void -LogMgr::Setup() { - el::Loggers::reconfigureLogger("default", el_config_); -} - -void -LogMgr::rename_and_delete(const std::string& filename, int64_t idx) { - std::string target_filename = filename + "." + std::to_string(idx); - rename(filename.c_str(), target_filename.c_str()); - if (enable_log_delete && idx - logs_delete_exceeds > 0) { - std::string to_delete = filename + "." + std::to_string(trace_idx - logs_delete_exceeds); - boost::filesystem::remove(to_delete); - } -} - -std::unordered_map -LogMgr::parse_level(const std::string& level) { - std::unordered_map enables{ - {"debug", false}, {"info", false}, {"warning", false}, {"error", false}, {"fatal", false}, - }; - - std::unordered_map level_to_int{ - {"debug", 5}, {"info", 4}, {"warning", 3}, {"error", 2}, {"fatal", 1}, - }; - - switch (level_to_int[level]) { - case 5: - enables["debug"] = true; - case 4: - enables["info"] = true; - case 3: - enables["warning"] = true; - case 2: - enables["error"] = true; - case 1: - enables["fatal"] = true; - break; - default: { - std::string msg = "Cannot parse level " + level + - ": invalid log level, must be one of debug, info, warning, error, fatal."; - throw std::runtime_error(msg); - } - } - return enables; -} - -void -LogMgr::enable(el::Configurations& default_conf, el::Level level, bool enable) { - std::string enable_str = enable ? "true" : "false"; - default_conf.set(level, el::ConfigurationType::Enabled, enable_str); -} - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/log/LogMgr.h b/internal/core/src/index/knowhere/log/LogMgr.h deleted file mode 100644 index d27d35f7f7..0000000000 --- a/internal/core/src/index/knowhere/log/LogMgr.h +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include "easyloggingpp/easylogging++.h" -#include "utils/Status.h" -#include "value/config/ServerConfig.h" - -#include -#include -#include - -namespace milvus { - -class LogMgr { - public: - static Status - InitLog(bool trace_enable, - const std::string& level, - const std::string& logs_path, - const std::string& filename, - int64_t max_log_file_size, - int64_t delete_exceeds, - bool log_to_stdout, - bool log_to_file); - - static void - RolloutHandler(const char* filename, std::size_t size, el::Level level); - - private: - explicit LogMgr(std::string log_path); - - LogMgr& - Default(); - - LogMgr& - Filename(const std::string& filename); - - /* Non-const for fiu to injecting error */ - LogMgr& - Level(std::unordered_map& enables); - - LogMgr& - To(bool log_to_stdout, bool log_to_file); - - LogMgr& - Rotate(int64_t max_log_file_size, int64_t log_rotate_num); - - void - Setup(); - - private: - static void - rename_and_delete(const std::string& filename, int64_t idx); - - static std::unordered_map - parse_level(const std::string& level); - - /** - * @brief Configures if output corresponding level log - */ - static void - enable(el::Configurations& default_conf, el::Level level, bool enable); - - private: - el::Configurations el_config_; - std::string logs_path_; - - private: - static int trace_idx; - static int global_idx; - static int debug_idx; - static int info_idx; - static int warning_idx; - static int error_idx; - static int fatal_idx; - static int64_t logs_delete_exceeds; - static bool enable_log_delete; - - const int64_t MAX_LOG_FILE_SIZE_MIN = 536870912; /* 512 MB */ - const int64_t MAX_LOG_FILE_SIZE_MAX = 4294967296; /* 4 GB */ - const int64_t LOG_ROTATE_NUM_MIN = 0; - const int64_t LOG_ROTATE_NUM_MAX = 1024; -}; - -} // namespace milvus diff --git a/internal/core/src/segcore/CMakeLists.txt b/internal/core/src/segcore/CMakeLists.txt index d0dca385d1..5b34289742 100644 --- a/internal/core/src/segcore/CMakeLists.txt +++ b/internal/core/src/segcore/CMakeLists.txt @@ -40,7 +40,6 @@ target_link_libraries(milvus_segcore milvus_common milvus_query milvus_utils - milvus_storage milvus_config ) diff --git a/internal/core/src/storage/CMakeLists.txt b/internal/core/src/storage/CMakeLists.txt deleted file mode 100644 index 287d2bccf1..0000000000 --- a/internal/core/src/storage/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License - -set(MILVUS_STORAGE_SRCS - StorageCache.cpp - ) - -add_library(milvus_storage ${MILVUS_STORAGE_SRCS}) -target_link_libraries(milvus_storage milvus_proto milvus_utils knowhere boost_bitset_ext) diff --git a/internal/core/src/storage/StorageCache.cpp b/internal/core/src/storage/StorageCache.cpp deleted file mode 100644 index b97e098c19..0000000000 --- a/internal/core/src/storage/StorageCache.cpp +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License - -#include "storage/StorageCache.h" diff --git a/internal/core/src/storage/StorageCache.h b/internal/core/src/storage/StorageCache.h deleted file mode 100644 index f0775ae19a..0000000000 --- a/internal/core/src/storage/StorageCache.h +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License - -#pragma once - -#include -#include "common/Types.h" - -namespace milvus::storage { - -// struct DataBlock { -// // TODO -// }; - -// struct BlockId { -// int64_t segment_id_; -// FieldId field_id_; -// }; - -// class StorageCacheEngine { -// // TODO -// }; - -} // namespace milvus::storage