From fa01a888a0f0b960c676aa9e79efcfeae2c9ea70 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Tue, 26 Oct 2021 17:26:30 +0800 Subject: [PATCH] Remove redundant files under internal/core (#10658) Signed-off-by: yudong.cai --- .../knowhere/archive/KnowhereConfig.cpp | 28 +- .../src/index/knowhere/utils/BlockingQueue.h | 95 ------ .../src/index/knowhere/utils/CommonUtil.cpp | 243 ------------- .../src/index/knowhere/utils/CommonUtil.h | 62 ---- .../src/index/knowhere/utils/ConfigUtils.cpp | 320 ------------------ .../src/index/knowhere/utils/ConfigUtils.h | 63 ---- .../core/src/index/knowhere/utils/Error.h | 145 -------- .../core/src/index/knowhere/utils/Exception.h | 61 ---- internal/core/src/index/knowhere/utils/Json.h | 24 +- .../index/knowhere/utils/SignalHandler.cpp | 60 ---- .../src/index/knowhere/utils/SignalHandler.h | 28 -- .../core/src/index/knowhere/utils/Status.cpp | 127 ------- .../core/src/index/knowhere/utils/Status.h | 69 +--- .../knowhere/utils/StringHelpFunctions.cpp | 175 ---------- .../knowhere/utils/StringHelpFunctions.h | 76 ----- .../src/index/knowhere/utils/ThreadPool.h | 128 ------- .../src/index/knowhere/utils/TimeRecorder.cpp | 93 ----- .../src/index/knowhere/utils/TimeRecorder.h | 65 ---- .../src/index/knowhere/utils/TimerContext.h | 76 ----- .../src/index/knowhere/utils/TimerManager.cpp | 85 ----- .../src/index/knowhere/utils/TimerManager.h | 57 ---- .../core/src/index/knowhere/utils/Types.cpp | 12 - .../core/src/index/knowhere/utils/Types.h | 51 --- .../core/src/index/knowhere/utils/tools.h | 30 -- internal/core/src/utils/BlockingQueue.h | 95 ------ internal/core/src/utils/CommonUtil.cpp | 243 ------------- internal/core/src/utils/CommonUtil.h | 62 ---- internal/core/src/utils/ConfigUtils.cpp | 320 ------------------ internal/core/src/utils/ConfigUtils.h | 63 ---- internal/core/src/utils/Exception.h | 61 ---- internal/core/src/utils/SignalHandler.cpp | 60 ---- internal/core/src/utils/SignalHandler.h | 28 -- internal/core/src/utils/Status.h | 2 +- .../core/src/utils/StringHelpFunctions.cpp | 175 ---------- internal/core/src/utils/StringHelpFunctions.h | 76 ----- internal/core/src/utils/ThreadPool.h | 128 ------- internal/core/src/utils/TimeRecorder.cpp | 93 ----- internal/core/src/utils/TimeRecorder.h | 65 ---- internal/core/src/utils/TimerContext.h | 76 ----- internal/core/src/utils/TimerManager.cpp | 85 ----- internal/core/src/utils/TimerManager.h | 57 ---- 41 files changed, 14 insertions(+), 3848 deletions(-) delete mode 100644 internal/core/src/index/knowhere/utils/BlockingQueue.h delete mode 100644 internal/core/src/index/knowhere/utils/CommonUtil.cpp delete mode 100644 internal/core/src/index/knowhere/utils/CommonUtil.h delete mode 100644 internal/core/src/index/knowhere/utils/ConfigUtils.cpp delete mode 100644 internal/core/src/index/knowhere/utils/ConfigUtils.h delete mode 100644 internal/core/src/index/knowhere/utils/Error.h delete mode 100644 internal/core/src/index/knowhere/utils/Exception.h delete mode 100644 internal/core/src/index/knowhere/utils/SignalHandler.cpp delete mode 100644 internal/core/src/index/knowhere/utils/SignalHandler.h delete mode 100644 internal/core/src/index/knowhere/utils/Status.cpp delete mode 100644 internal/core/src/index/knowhere/utils/StringHelpFunctions.cpp delete mode 100644 internal/core/src/index/knowhere/utils/StringHelpFunctions.h delete mode 100644 internal/core/src/index/knowhere/utils/ThreadPool.h delete mode 100644 internal/core/src/index/knowhere/utils/TimeRecorder.cpp delete mode 100644 internal/core/src/index/knowhere/utils/TimeRecorder.h delete mode 100644 internal/core/src/index/knowhere/utils/TimerContext.h delete mode 100644 internal/core/src/index/knowhere/utils/TimerManager.cpp delete mode 100644 internal/core/src/index/knowhere/utils/TimerManager.h delete mode 100644 internal/core/src/index/knowhere/utils/Types.cpp delete mode 100644 internal/core/src/index/knowhere/utils/Types.h delete mode 100644 internal/core/src/index/knowhere/utils/tools.h delete mode 100644 internal/core/src/utils/BlockingQueue.h delete mode 100644 internal/core/src/utils/CommonUtil.cpp delete mode 100644 internal/core/src/utils/CommonUtil.h delete mode 100644 internal/core/src/utils/ConfigUtils.cpp delete mode 100644 internal/core/src/utils/ConfigUtils.h delete mode 100644 internal/core/src/utils/Exception.h delete mode 100644 internal/core/src/utils/SignalHandler.cpp delete mode 100644 internal/core/src/utils/SignalHandler.h delete mode 100644 internal/core/src/utils/StringHelpFunctions.cpp delete mode 100644 internal/core/src/utils/StringHelpFunctions.h delete mode 100644 internal/core/src/utils/ThreadPool.h delete mode 100644 internal/core/src/utils/TimeRecorder.cpp delete mode 100644 internal/core/src/utils/TimeRecorder.h delete mode 100644 internal/core/src/utils/TimerContext.h delete mode 100644 internal/core/src/utils/TimerManager.cpp delete mode 100644 internal/core/src/utils/TimerManager.h diff --git a/internal/core/src/index/knowhere/knowhere/archive/KnowhereConfig.cpp b/internal/core/src/index/knowhere/knowhere/archive/KnowhereConfig.cpp index c71b627ceb..eb38b26556 100644 --- a/internal/core/src/index/knowhere/knowhere/archive/KnowhereConfig.cpp +++ b/internal/core/src/index/knowhere/knowhere/archive/KnowhereConfig.cpp @@ -9,27 +9,21 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. +#include + #include "KnowhereConfig.h" +#include "NGT/lib/NGT/defines.h" +#include "faiss/Clustering.h" +#include "faiss/FaissHook.h" +#include "faiss/common.h" +#include "faiss/utils/distances.h" +#include "faiss/utils/utils.h" +#include "knowhere/common/Exception.h" +#include "knowhere/common/Log.h" +#include "knowhere/index/vector_index/IndexHNSW.h" #ifdef MILVUS_GPU_VERSION #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #endif -#include -#include - -#include "NGT/lib/NGT/defines.h" -#include "faiss/FaissHook.h" -#include "faiss/common.h" -#include "faiss/utils/utils.h" -#include "knowhere/common/Log.h" -#include "knowhere/index/IndexType.h" -#include "knowhere/index/vector_index/IndexHNSW.h" -#include "knowhere/index/vector_index/helpers/FaissIO.h" -#include "utils/ConfigUtils.h" -#include "utils/Error.h" -#include "knowhere/common/Exception.h" - -#include -#include namespace milvus { namespace engine { diff --git a/internal/core/src/index/knowhere/utils/BlockingQueue.h b/internal/core/src/index/knowhere/utils/BlockingQueue.h deleted file mode 100644 index 1b489f53e6..0000000000 --- a/internal/core/src/index/knowhere/utils/BlockingQueue.h +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include - -namespace milvus { - -template -class BlockingQueue { - public: - BlockingQueue() : mtx(), full_(), empty_() { - } - - virtual ~BlockingQueue() { - } - - BlockingQueue(const BlockingQueue& rhs) = delete; - - BlockingQueue& - operator=(const BlockingQueue& rhs) = delete; - - void - Put(const T& task) { - std::unique_lock lock(mtx); - full_.wait(lock, [this] { return (queue_.size() < capacity_); }); - queue_.push(task); - empty_.notify_all(); - } - - T - Take() { - std::unique_lock lock(mtx); - empty_.wait(lock, [this] { return !queue_.empty(); }); - T front(queue_.front()); - queue_.pop(); - full_.notify_all(); - return front; - } - - T - Front() { - std::unique_lock lock(mtx); - empty_.wait(lock, [this] { return !queue_.empty(); }); - T front(queue_.front()); - return front; - } - - T - Back() { - std::unique_lock lock(mtx); - empty_.wait(lock, [this] { return !queue_.empty(); }); - T back(queue_.back()); - return back; - } - - size_t - Size() const { - std::lock_guard lock(mtx); - return queue_.size(); - } - - bool - Empty() const { - std::unique_lock lock(mtx); - return queue_.empty(); - } - - void - SetCapacity(const size_t capacity) { - capacity_ = (capacity > 0 ? capacity : capacity_); - } - - protected: - mutable std::mutex mtx; - std::condition_variable full_; - std::condition_variable empty_; - std::queue queue_; - size_t capacity_ = 32; -}; - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/CommonUtil.cpp b/internal/core/src/index/knowhere/utils/CommonUtil.cpp deleted file mode 100644 index f2b4215972..0000000000 --- a/internal/core/src/index/knowhere/utils/CommonUtil.cpp +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "utils/CommonUtil.h" -#include "utils/Log.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace milvus { - -namespace fs = boost::filesystem; - -bool -CommonUtil::IsDirectoryExist(const std::string& path) { - DIR* dp = nullptr; - if ((dp = opendir(path.c_str())) == nullptr) { - return false; - } - - closedir(dp); - return true; -} - -Status -CommonUtil::CreateDirectory(const std::string& path) { - if (path.empty()) { - return Status::OK(); - } - - struct stat directory_stat; - int status = stat(path.c_str(), &directory_stat); - if (status == 0) { - return Status::OK(); // already exist - } - - fs::path fs_path(path); - fs::path parent_path = fs_path.parent_path(); - Status err_status = CreateDirectory(parent_path.string()); - fiu_do_on("CommonUtil.CreateDirectory.create_parent_fail", err_status = Status(SERVER_INVALID_ARGUMENT, "")); - if (!err_status.ok()) { - return err_status; - } - - status = stat(path.c_str(), &directory_stat); - if (status == 0) { - return Status::OK(); // already exist - } - - int makeOK = mkdir(path.c_str(), S_IRWXU | S_IRGRP | S_IROTH); - fiu_do_on("CommonUtil.CreateDirectory.create_dir_fail", makeOK = 1); - if (makeOK != 0) { - return Status(SERVER_UNEXPECTED_ERROR, "failed to create directory: " + path); - } - - return Status::OK(); -} - -namespace { -void -RemoveDirectory(const std::string& path) { - DIR* dir = nullptr; - const int32_t buf_size = 256; - char file_name[buf_size]; - - std::string folder_name = path + "/%s"; - if ((dir = opendir(path.c_str())) != nullptr) { - struct dirent* dmsg; - while ((dmsg = readdir(dir)) != nullptr) { - if (strcmp(dmsg->d_name, ".") != 0 && strcmp(dmsg->d_name, "..") != 0) { - snprintf(file_name, buf_size, folder_name.c_str(), dmsg->d_name); - std::string tmp = file_name; - if (tmp.find('.') == std::string::npos) { - RemoveDirectory(file_name); - } - remove(file_name); - } - } - } - - if (dir != nullptr) { - closedir(dir); - } - remove(path.c_str()); -} -} // namespace - -Status -CommonUtil::DeleteDirectory(const std::string& path) { - if (path.empty()) { - return Status::OK(); - } - - struct stat directory_stat; - int statOK = stat(path.c_str(), &directory_stat); - if (statOK != 0) { - return Status::OK(); - } - - RemoveDirectory(path); - return Status::OK(); -} - -bool -CommonUtil::IsFileExist(const std::string& path) { - return (access(path.c_str(), F_OK) == 0); -} - -uint64_t -CommonUtil::GetFileSize(const std::string& path) { - struct stat file_info; - if (stat(path.c_str(), &file_info) < 0) { - return 0; - } - - return static_cast(file_info.st_size); -} - -std::string -CommonUtil::GetFileName(std::string filename) { - int pos = filename.find_last_of('/'); - return filename.substr(pos + 1); -} - -std::string -CommonUtil::GetExePath() { - const int64_t buf_len = 1024; - char buf[buf_len]; - int64_t cnt = readlink("/proc/self/exe", buf, buf_len); - fiu_do_on("CommonUtil.GetExePath.readlink_fail", cnt = -1); - if (cnt < 0 || cnt >= buf_len) { - return ""; - } - - buf[cnt] = '\0'; - - std::string exe_path = buf; - fiu_do_on("CommonUtil.GetExePath.exe_path_error", exe_path = "/"); - if (exe_path.rfind('/') != exe_path.length() - 1) { - std::string sub_str = exe_path.substr(0, exe_path.rfind('/')); - return sub_str + "/"; - } - return exe_path; -} - -bool -CommonUtil::TimeStrToTime(const std::string& time_str, - time_t& time_integer, - tm& time_struct, - const std::string& format) { - time_integer = 0; - memset(&time_struct, 0, sizeof(tm)); - - int ret = sscanf(time_str.c_str(), format.c_str(), &(time_struct.tm_year), &(time_struct.tm_mon), - &(time_struct.tm_mday), &(time_struct.tm_hour), &(time_struct.tm_min), &(time_struct.tm_sec)); - if (ret <= 0) { - return false; - } - - time_struct.tm_year -= 1900; - time_struct.tm_mon--; - time_integer = mktime(&time_struct); - - return true; -} - -void -CommonUtil::GetCurrentTimeStr(std::string& time_str) { - auto t = std::time(nullptr); - struct tm ltm; - localtime_r(&t, <m); - - time_str = ""; - time_str += std::to_string(ltm.tm_year + 1900); - time_str += "-"; - time_str += std::to_string(ltm.tm_mon + 1); - time_str += "-"; - time_str += std::to_string(ltm.tm_mday); - time_str += "_"; - time_str += std::to_string(ltm.tm_hour); - time_str += ":"; - time_str += std::to_string(ltm.tm_min); - time_str += ":"; - time_str += std::to_string(ltm.tm_sec); -} - -void -CommonUtil::ConvertTime(time_t time_integer, tm& time_struct) { - localtime_r(&time_integer, &time_struct); -} - -void -CommonUtil::ConvertTime(tm time_struct, time_t& time_integer) { - time_integer = mktime(&time_struct); -} - -std::string -CommonUtil::ConvertSize(int64_t size) { - const int64_t gb = 1024ll * 1024 * 1024; - const int64_t mb = 1024ll * 1024; - const int64_t kb = 1024ll; - if (size % gb == 0) { - return std::to_string(size / gb) + "GB"; - } else if (size % mb == 0) { - return std::to_string(size / mb) + "MB"; - } else if (size % kb == 0) { - return std::to_string(size / kb) + "KB"; - } else { - return std::to_string(size); - } -} - -#ifdef ENABLE_CPU_PROFILING -std::string -CommonUtil::GetCurrentTimeStr() { - time_t tt; - time(&tt); - tt = tt + 8 * 60; - tm t; - gmtime_r(&tt, &t); - - std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) + "_" + - std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) + "_" + std::to_string(t.tm_min) + - "_" + std::to_string(t.tm_sec); - return str; -} -#endif - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/CommonUtil.h b/internal/core/src/index/knowhere/utils/CommonUtil.h deleted file mode 100644 index 4da0a30505..0000000000 --- a/internal/core/src/index/knowhere/utils/CommonUtil.h +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include "utils/Status.h" - -#include -#include - -namespace milvus { - -class CommonUtil { - public: - static bool - IsFileExist(const std::string& path); - static uint64_t - GetFileSize(const std::string& path); - static bool - IsDirectoryExist(const std::string& path); - static Status - CreateDirectory(const std::string& path); - static Status - DeleteDirectory(const std::string& path); - - static std::string - GetFileName(std::string filename); - static std::string - GetExePath(); - - static bool - TimeStrToTime(const std::string& time_str, - time_t& time_integer, - tm& time_struct, - const std::string& format = "%d-%d-%d %d:%d:%d"); - - static void - GetCurrentTimeStr(std::string& time_str); - - static void - ConvertTime(time_t time_integer, tm& time_struct); - static void - ConvertTime(tm time_struct, time_t& time_integer); - - static std::string - ConvertSize(int64_t size); - -#ifdef ENABLE_CPU_PROFILING - static std::string - GetCurrentTimeStr(); -#endif -}; - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/ConfigUtils.cpp b/internal/core/src/index/knowhere/utils/ConfigUtils.cpp deleted file mode 100644 index 8f459954d4..0000000000 --- a/internal/core/src/index/knowhere/utils/ConfigUtils.cpp +++ /dev/null @@ -1,320 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "utils/ConfigUtils.h" -#include "utils/Log.h" -#include "utils/StringHelpFunctions.h" - -#include -#include -#include -#ifdef MILVUS_GPU_VERSION -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(__x86_64__) -#define THREAD_MULTIPLY_CPU 1 -#elif defined(__powerpc64__) -#define THREAD_MULTIPLY_CPU 4 -#else -#define THREAD_MULTIPLY_CPU 1 -#endif - -namespace milvus { -namespace server { - -std::unordered_map BYTE_UNITS = { - {"b", 1}, - {"k", 1024}, - {"m", 1024 * 1024}, - {"g", 1024 * 1024 * 1024}, -}; - -bool -is_number(const std::string& s) { - return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isdigit(c); }) == s.end(); -} - -bool -is_alpha(const std::string& s) { - return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isalpha(c); }) == s.end(); -} - -std::string -str_tolower(std::string s) { - std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); }); - return s; -} - -int64_t -parse_bytes(const std::string& str, std::string& err) { - try { - std::string s = str; - if (is_number(s)) { - return std::stoll(s); - } - if (s.length() == 0) { - return 0; - } - - auto last_two = s.substr(s.length() - 2, 2); - auto last_one = s.substr(s.length() - 1); - if (is_alpha(last_two) && is_alpha(last_one)) { - if (last_one == "b" or last_one == "B") { - s = s.substr(0, s.length() - 1); - } - } - auto& units = BYTE_UNITS; - auto suffix = str_tolower(s.substr(s.length() - 1)); - - std::string digits_part; - if (is_number(suffix)) { - digits_part = s; - suffix = 'b'; - } else { - digits_part = s.substr(0, s.length() - 1); - } - - if (units.find(suffix) != units.end() or is_number(suffix)) { - auto digits = std::stoll(digits_part); - return digits * units[suffix]; - } else { - std::stringstream ss; - ss << "The specified value for memory (" << str << ") should specify the units." - << "The postfix should be one of the `b` `k` `m` `g` characters"; - err = ss.str(); - } - } catch (...) { - err = "Unknown error happened on parse bytes."; - } - return 0; -} - -bool -GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem) { - struct sysinfo info; - int ret = sysinfo(&info); - total_mem = info.totalram; - free_mem = info.freeram; - - return ret == 0; // succeed 0, failed -1 -} - -bool -GetSystemAvailableThreads(int64_t& thread_count) { - // threadCnt = std::thread::hardware_concurrency(); - thread_count = sysconf(_SC_NPROCESSORS_CONF); - thread_count *= THREAD_MULTIPLY_CPU; - fiu_do_on("GetSystemAvailableThreads.zero_thread", thread_count = 0); - - if (thread_count == 0) { - thread_count = 8; - } - - return true; -} - -Status -ValidateGpuIndex(int32_t gpu_index) { -#ifdef MILVUS_GPU_VERSION - int num_devices = 0; - auto cuda_err = cudaGetDeviceCount(&num_devices); - fiu_do_on("config.ValidateGpuIndex.get_device_count_fail", cuda_err = cudaError::cudaErrorUnknown); - - if (cuda_err != cudaSuccess) { - std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err); - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_UNEXPECTED_ERROR, msg); - } - - if (gpu_index >= num_devices) { - std::string msg = "Invalid gpu index: " + std::to_string(gpu_index); - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - } -#endif - - return Status::OK(); -} - -#ifdef MILVUS_GPU_VERSION -Status -GetGpuMemory(int32_t gpu_index, int64_t& memory) { - fiu_return_on("config.GetGpuMemory.return_error", Status(SERVER_UNEXPECTED_ERROR, "")); - - cudaDeviceProp deviceProp; - auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index); - if (cuda_err) { - std::string msg = "Failed to get gpu properties for gpu" + std::to_string(gpu_index) + - " , cuda error:" + std::to_string(cuda_err); - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_UNEXPECTED_ERROR, msg); - } - - memory = deviceProp.totalGlobalMem; - return Status::OK(); -} -#endif - -Status -ValidateIpAddress(const std::string& ip_address) { - struct in_addr address; - - int result = inet_pton(AF_INET, ip_address.c_str(), &address); - fiu_do_on("config.ValidateIpAddress.error_ip_result", result = 2); - - switch (result) { - case 1: - return Status::OK(); - case 0: { - std::string msg = "Invalid IP address: " + ip_address; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - default: { - std::string msg = "IP address conversion error: " + ip_address; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_UNEXPECTED_ERROR, msg); - } - } -} - -Status -ValidateStringIsNumber(const std::string& str) { - if (str.empty() || !std::all_of(str.begin(), str.end(), ::isdigit)) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid number"); - } - try { - int64_t value = std::stol(str); - fiu_do_on("config.ValidateStringIsNumber.throw_exception", throw std::exception()); - if (value < 0) { - return Status(SERVER_INVALID_ARGUMENT, "Negative number"); - } - } catch (...) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid number"); - } - return Status::OK(); -} - -Status -ValidateStringIsBool(const std::string& str) { - fiu_return_on("ValidateStringNotBool", Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str)); - std::string s = str; - std::transform(s.begin(), s.end(), s.begin(), ::tolower); - if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" || - s.empty()) { - return Status::OK(); - } - return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str); -} - -Status -ValidateStringIsFloat(const std::string& str) { - try { - float val = std::stof(str); - if (val < 0.0) { - return Status(SERVER_INVALID_ARGUMENT, "Negative float: " + str); - } - } catch (...) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str); - } - return Status::OK(); -} - -Status -ValidateDbURI(const std::string& uri) { - std::string dialectRegex = "(.*)"; - std::string usernameRegex = "(.*)"; - std::string passwordRegex = "(.*)"; - std::string hostRegex = "(.*)"; - std::string portRegex = "(.*)"; - std::string dbNameRegex = "(.*)"; - std::string uriRegexStr = dialectRegex + R"(\:\/\/)" + usernameRegex + R"(\:)" + passwordRegex + R"(\@)" + - hostRegex + R"(\:)" + portRegex + R"(\/)" + dbNameRegex; - std::regex uriRegex(uriRegexStr); - std::smatch pieces_match; - - bool okay = true; - - if (std::regex_match(uri, pieces_match, uriRegex)) { - std::string dialect = pieces_match[1].str(); - std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); - if (dialect.find("mysql") == std::string::npos && dialect.find("sqlite") == std::string::npos && - dialect.find("mock") == std::string::npos) { - LOG_SERVER_ERROR_ << "Invalid dialect in URI: dialect = " << dialect; - okay = false; - } - - /* - * Could be DNS, skip checking - * - std::string host = pieces_match[4].str(); - if (!host.empty() && host != "localhost") { - if (ValidateIpAddress(host) != SERVER_SUCCESS) { - LOG_SERVER_ERROR_ << "Invalid host ip address in uri = " << host; - okay = false; - } - } - */ - - std::string port = pieces_match[5].str(); - if (!port.empty()) { - auto status = ValidateStringIsNumber(port); - if (!status.ok()) { - LOG_SERVER_ERROR_ << "Invalid port in uri = " << port; - okay = false; - } - } - } else { - LOG_SERVER_ERROR_ << "Wrong URI format: URI = " << uri; - okay = false; - } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid db backend uri")); -} - -Status -ValidateStoragePath(const std::string& path) { - // Validate storage path if is valid, only correct absolute path will be validated pass - // Invalid path only contain character[a-zA-Z], number[0-9], '-', and '_', - // and path must start with '/'. - // examples below are invalid - // '/a//a', '/a--/a', '/-a/a', '/a@#/a', 'aaa/sfs' - std::string path_pattern = R"(^\/(\w+-?\/?)+$)"; - std::regex regex(path_pattern); - - return std::regex_match(path, regex) ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid file path"); -} - -Status -ValidateLogLevel(const std::string& level) { - std::set supported_level{"debug", "info", "warning", "error", "fatal"}; - - return supported_level.find(level) != supported_level.end() - ? Status::OK() - : Status(SERVER_INVALID_ARGUMENT, "Log level must be one of debug, info, warning, error and fatal."); -} - -bool -IsNumber(const std::string& s) { - return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit); -} - -} // namespace server -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/ConfigUtils.h b/internal/core/src/index/knowhere/utils/ConfigUtils.h deleted file mode 100644 index 27d3ce6842..0000000000 --- a/internal/core/src/index/knowhere/utils/ConfigUtils.h +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include - -#include "utils/Status.h" - -namespace milvus { -namespace server { - -extern int64_t -parse_bytes(const std::string& str, std::string& err); - -extern bool -GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem); - -extern bool -GetSystemAvailableThreads(int64_t& thread_count); - -extern Status -ValidateGpuIndex(int32_t gpu_index); - -#ifdef MILVUS_GPU_VERSION -extern Status -GetGpuMemory(int32_t gpu_index, int64_t& memory); -#endif - -extern Status -ValidateIpAddress(const std::string& ip_address); - -extern Status -ValidateStringIsNumber(const std::string& str); - -extern Status -ValidateStringIsBool(const std::string& str); - -extern Status -ValidateStringIsFloat(const std::string& str); - -extern Status -ValidateDbURI(const std::string& uri); - -extern Status -ValidateStoragePath(const std::string& path); - -extern Status -ValidateLogLevel(const std::string& level); - -extern bool -IsNumber(const std::string& s); -} // namespace server -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/Error.h b/internal/core/src/index/knowhere/utils/Error.h deleted file mode 100644 index 7f1eeed86d..0000000000 --- a/internal/core/src/index/knowhere/utils/Error.h +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include - -namespace milvus { - -using ErrorCode = int32_t; - -constexpr ErrorCode SERVER_SUCCESS = 0; -constexpr ErrorCode SERVER_ERROR_CODE_BASE = 30000; - -constexpr ErrorCode -ToServerErrorCode(const ErrorCode error_code) { - return SERVER_ERROR_CODE_BASE + error_code; -} - -constexpr ErrorCode DB_SUCCESS = 0; -constexpr ErrorCode DB_ERROR_CODE_BASE = 40000; - -constexpr ErrorCode -ToDbErrorCode(const ErrorCode error_code) { - return DB_ERROR_CODE_BASE + error_code; -} - -constexpr ErrorCode KNOWHERE_SUCCESS = 0; -constexpr ErrorCode KNOWHERE_ERROR_CODE_BASE = 50000; - -constexpr ErrorCode -ToKnowhereErrorCode(const ErrorCode error_code) { - return KNOWHERE_ERROR_CODE_BASE + error_code; -} - -constexpr ErrorCode WAL_SUCCESS = 0; -constexpr ErrorCode WAL_ERROR_CODE_BASE = 60000; - -constexpr ErrorCode -ToWalErrorCode(const ErrorCode error_code) { - return WAL_ERROR_CODE_BASE + error_code; -} - -constexpr ErrorCode SS_SUCCESS = 0; -constexpr ErrorCode SS_ERROR_CODE_BASE = 70000; - -constexpr ErrorCode -ToSSErrorCode(const ErrorCode error_code) { - return SS_ERROR_CODE_BASE + error_code; -} - -// server error code -constexpr ErrorCode SERVER_UNEXPECTED_ERROR = ToServerErrorCode(1); -constexpr ErrorCode SERVER_UNSUPPORTED_ERROR = ToServerErrorCode(2); -constexpr ErrorCode SERVER_NULL_POINTER = ToServerErrorCode(3); -constexpr ErrorCode SERVER_INVALID_ARGUMENT = ToServerErrorCode(4); -constexpr ErrorCode SERVER_FILE_NOT_FOUND = ToServerErrorCode(5); -constexpr ErrorCode SERVER_NOT_IMPLEMENT = ToServerErrorCode(6); -constexpr ErrorCode SERVER_CANNOT_CREATE_FOLDER = ToServerErrorCode(8); -constexpr ErrorCode SERVER_CANNOT_CREATE_FILE = ToServerErrorCode(9); -constexpr ErrorCode SERVER_CANNOT_DELETE_FOLDER = ToServerErrorCode(10); -constexpr ErrorCode SERVER_CANNOT_DELETE_FILE = ToServerErrorCode(11); -constexpr ErrorCode SERVER_BUILD_INDEX_ERROR = ToServerErrorCode(12); -constexpr ErrorCode SERVER_CANNOT_OPEN_FILE = ToServerErrorCode(13); -constexpr ErrorCode SERVER_FILE_MAGIC_BYTES_ERROR = ToServerErrorCode(14); -constexpr ErrorCode SERVER_FILE_SUM_BYTES_ERROR = ToServerErrorCode(15); -constexpr ErrorCode SERVER_CANNOT_READ_FILE = ToServerErrorCode(16); - -constexpr ErrorCode SERVER_COLLECTION_NOT_EXIST = ToServerErrorCode(100); -constexpr ErrorCode SERVER_INVALID_COLLECTION_NAME = ToServerErrorCode(101); -constexpr ErrorCode SERVER_INVALID_COLLECTION_DIMENSION = ToServerErrorCode(102); -constexpr ErrorCode SERVER_INVALID_VECTOR_DIMENSION = ToServerErrorCode(104); -constexpr ErrorCode SERVER_INVALID_INDEX_TYPE = ToServerErrorCode(105); -constexpr ErrorCode SERVER_INVALID_ROWRECORD = ToServerErrorCode(106); -constexpr ErrorCode SERVER_INVALID_ROWRECORD_ARRAY = ToServerErrorCode(107); -constexpr ErrorCode SERVER_INVALID_TOPK = ToServerErrorCode(108); -constexpr ErrorCode SERVER_ILLEGAL_VECTOR_ID = ToServerErrorCode(109); -constexpr ErrorCode SERVER_ILLEGAL_SEARCH_RESULT = ToServerErrorCode(110); -constexpr ErrorCode SERVER_CACHE_FULL = ToServerErrorCode(111); -constexpr ErrorCode SERVER_WRITE_ERROR = ToServerErrorCode(112); -constexpr ErrorCode SERVER_INVALID_NPROBE = ToServerErrorCode(113); -constexpr ErrorCode SERVER_INVALID_INDEX_NLIST = ToServerErrorCode(114); -constexpr ErrorCode SERVER_INVALID_INDEX_METRIC_TYPE = ToServerErrorCode(115); -constexpr ErrorCode SERVER_INVALID_SEGMENT_ROW_COUNT = ToServerErrorCode(116); -constexpr ErrorCode SERVER_OUT_OF_MEMORY = ToServerErrorCode(117); -constexpr ErrorCode SERVER_INVALID_PARTITION_TAG = ToServerErrorCode(118); -constexpr ErrorCode SERVER_INVALID_BINARY_QUERY = ToServerErrorCode(119); -constexpr ErrorCode SERVER_INVALID_DSL_PARAMETER = ToServerErrorCode(120); -constexpr ErrorCode SERVER_INVALID_FIELD_NAME = ToServerErrorCode(121); -constexpr ErrorCode SERVER_INVALID_FIELD_NUM = ToServerErrorCode(122); - -// db error code -constexpr ErrorCode DB_META_TRANSACTION_FAILED = ToDbErrorCode(1); -constexpr ErrorCode DB_ERROR = ToDbErrorCode(2); -constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3); -constexpr ErrorCode DB_ALREADY_EXIST = ToDbErrorCode(4); -constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5); -constexpr ErrorCode DB_INCOMPATIB_META = ToDbErrorCode(6); -constexpr ErrorCode DB_INVALID_META_URI = ToDbErrorCode(7); -constexpr ErrorCode DB_EMPTY_COLLECTION = ToDbErrorCode(8); -constexpr ErrorCode DB_BLOOM_FILTER_ERROR = ToDbErrorCode(9); -constexpr ErrorCode DB_PARTITION_NOT_FOUND = ToDbErrorCode(10); -constexpr ErrorCode DB_OUT_OF_STORAGE = ToDbErrorCode(11); -constexpr ErrorCode DB_META_QUERY_FAILED = ToDbErrorCode(12); -constexpr ErrorCode DB_FILE_NOT_FOUND = ToDbErrorCode(13); -constexpr ErrorCode DB_PERMISSION_ERROR = ToDbErrorCode(14); - -// knowhere error code -constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1); -constexpr ErrorCode KNOWHERE_INVALID_ARGUMENT = ToKnowhereErrorCode(2); -constexpr ErrorCode KNOWHERE_UNEXPECTED_ERROR = ToKnowhereErrorCode(3); -constexpr ErrorCode KNOWHERE_NO_SPACE = ToKnowhereErrorCode(4); - -// knowhere error code -constexpr ErrorCode WAL_ERROR = ToWalErrorCode(1); -constexpr ErrorCode WAL_META_ERROR = ToWalErrorCode(2); -constexpr ErrorCode WAL_FILE_ERROR = ToWalErrorCode(3); -constexpr ErrorCode WAL_PATH_ERROR = ToWalErrorCode(4); - -// Snapshot error code -constexpr ErrorCode SS_ERROR = ToSSErrorCode(1); -constexpr ErrorCode SS_STALE_ERROR = ToSSErrorCode(2); -constexpr ErrorCode SS_NOT_FOUND_ERROR = ToSSErrorCode(3); -constexpr ErrorCode SS_INVALID_CONTEX_ERROR = ToSSErrorCode(4); -constexpr ErrorCode SS_DUPLICATED_ERROR = ToSSErrorCode(5); -constexpr ErrorCode SS_NOT_ACTIVE_ERROR = ToSSErrorCode(6); -constexpr ErrorCode SS_CONSTRAINT_CHECK_ERROR = ToSSErrorCode(7); -constexpr ErrorCode SS_INVALID_ARGUMENT_ERROR = ToSSErrorCode(8); -constexpr ErrorCode SS_OPERATION_PENDING = ToSSErrorCode(9); -constexpr ErrorCode SS_TIMEOUT = ToSSErrorCode(10); -constexpr ErrorCode SS_NOT_COMMITED = ToSSErrorCode(11); -constexpr ErrorCode SS_COLLECTION_DROPPED = ToSSErrorCode(12); -constexpr ErrorCode SS_EMPTY_HOLDER = ToSSErrorCode(13); - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/Exception.h b/internal/core/src/index/knowhere/utils/Exception.h deleted file mode 100644 index 454fcb508f..0000000000 --- a/internal/core/src/index/knowhere/utils/Exception.h +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include "utils/Error.h" - -#include -#include -#include - -namespace milvus { - -#define THROW_ERROR(err_code, err_msg) \ - LOG_ENGINE_ERROR_ << err_msg; \ - throw Exception(err_code, err_msg); - -class Exception : public std::exception { - public: - Exception(ErrorCode code, std::string msg) : code_(code), message_(std::move(msg)) { - } - - ErrorCode - code() const noexcept { - return code_; - } - - const char* - what() const noexcept override { - if (message_.empty()) { - return "Default Exception."; - } else { - return message_.c_str(); - } - } - - ~Exception() noexcept override = default; - - protected: - ErrorCode code_; - std::string message_; -}; - -class InvalidArgumentException : public Exception { - public: - InvalidArgumentException() : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") { - } - - explicit InvalidArgumentException(const std::string& message) : Exception(SERVER_INVALID_ARGUMENT, message) { - } -}; - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/Json.h b/internal/core/src/index/knowhere/utils/Json.h index 03ee2127fb..f74b7ae66a 100644 --- a/internal/core/src/index/knowhere/utils/Json.h +++ b/internal/core/src/index/knowhere/utils/Json.h @@ -9,26 +9,4 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. -#pragma once - -#include "nlohmann/json.hpp" - -namespace milvus { - -using json = nlohmann::json; - -#define JSON_NULL_CHECK(json) \ - do { \ - if (json.empty()) { \ - return Status{SERVER_INVALID_ARGUMENT, "Json is null"}; \ - } \ - } while (false) - -#define JSON_OBJECT_CHECK(json) \ - do { \ - if (!json.is_object()) { \ - return Status{SERVER_INVALID_ARGUMENT, "Json is not a json object"}; \ - } \ - } while (false) - -} // namespace milvus +#include "../../../utils/Json.h" diff --git a/internal/core/src/index/knowhere/utils/SignalHandler.cpp b/internal/core/src/index/knowhere/utils/SignalHandler.cpp deleted file mode 100644 index de234d795b..0000000000 --- a/internal/core/src/index/knowhere/utils/SignalHandler.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "utils/SignalHandler.h" -#include "utils/Log.h" - -#include -#include - -namespace milvus { - -signal_func_ptr signal_routine_func = nullptr; - -void -HandleSignal(int signum) { - int32_t exit_code = 1; /* 0: normal exit; 1: exception */ - switch (signum) { - case SIGINT: - case SIGUSR2: - exit_code = 0; - /* no break */ - default: { - if (exit_code == 0) { - LOG_SERVER_INFO_ << "Server received signal: " << signum; - } else { - LOG_SERVER_INFO_ << "Server received critical signal: " << signum; - PrintStacktrace(); - } - if (signal_routine_func != nullptr) { - (*signal_routine_func)(exit_code); - } - } - } -} - -void -PrintStacktrace() { - const int bt_depth = 128; - void* array[bt_depth]; - int stack_num = backtrace(array, bt_depth); - char** stacktrace = backtrace_symbols(array, stack_num); - - LOG_SERVER_INFO_ << "Call stack:"; - for (int i = 0; i < stack_num; ++i) { - std::string info = stacktrace[i]; - std::cout << "No." << i << ": " << info << std::endl; - LOG_SERVER_INFO_ << info; - } - free(stacktrace); -} - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/SignalHandler.h b/internal/core/src/index/knowhere/utils/SignalHandler.h deleted file mode 100644 index 5c58886e45..0000000000 --- a/internal/core/src/index/knowhere/utils/SignalHandler.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include - -namespace milvus { - -typedef void (*signal_func_ptr)(int32_t); - -extern signal_func_ptr signal_routine_func; - -extern void -HandleSignal(int signum); - -extern void -PrintStacktrace(); - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/Status.cpp b/internal/core/src/index/knowhere/utils/Status.cpp deleted file mode 100644 index 0fa2a2fea6..0000000000 --- a/internal/core/src/index/knowhere/utils/Status.cpp +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "utils/Status.h" -#include "memory" - -#include - -namespace milvus { - -constexpr int CODE_WIDTH = sizeof(StatusCode); - -Status::Status(StatusCode code, const std::string& msg) { - // 4 bytes store code - // 4 bytes store message length - // the left bytes store message string - auto length = static_cast(msg.size()); - // auto result = new char[length + sizeof(length) + CODE_WIDTH]; - state_.resize(length + sizeof(length) + CODE_WIDTH); - std::memcpy(state_.data(), &code, CODE_WIDTH); - std::memcpy(state_.data() + CODE_WIDTH, &length, sizeof(length)); - memcpy(state_.data() + sizeof(length) + CODE_WIDTH, msg.data(), length); -} - -Status::~Status() { -} - -Status::Status(const Status& s) { - CopyFrom(s); -} - -Status::Status(Status&& s) noexcept { - MoveFrom(s); -} - -Status& -Status::operator=(const Status& s) { - CopyFrom(s); - return *this; -} - -Status& -Status::operator=(Status&& s) noexcept { - MoveFrom(s); - return *this; -} - -void -Status::CopyFrom(const Status& s) { - state_.clear(); - if (s.state_.empty()) { - return; - } - - uint32_t length = 0; - memcpy(&length, s.state_.data() + CODE_WIDTH, sizeof(length)); - int buff_len = length + sizeof(length) + CODE_WIDTH; - state_.resize(buff_len); - memcpy(state_.data(), s.state_.data(), buff_len); -} - -void -Status::MoveFrom(Status& s) { - state_ = s.state_; - s.state_.clear(); -} - -std::string -Status::message() const { - if (state_.empty()) { - return "OK"; - } - - std::string msg; - uint32_t length = 0; - memcpy(&length, state_.data() + CODE_WIDTH, sizeof(length)); - if (length > 0) { - msg.append(state_.data() + sizeof(length) + CODE_WIDTH, length); - } - - return msg; -} - -std::string -Status::ToString() const { - if (state_.empty()) { - return "OK"; - } - - std::string result; - switch (code()) { - case DB_SUCCESS: - result = "OK "; - break; - case DB_ERROR: - result = "Error: "; - break; - case DB_META_TRANSACTION_FAILED: - result = "Database error: "; - break; - case DB_NOT_FOUND: - result = "Not found: "; - break; - case DB_ALREADY_EXIST: - result = "Already exist: "; - break; - case DB_INVALID_PATH: - result = "Invalid path: "; - break; - default: - result = "Error code(" + std::to_string(code()) + "): "; - break; - } - - result += message(); - return result; -} - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/Status.h b/internal/core/src/index/knowhere/utils/Status.h index 68342a3070..a49f27f5d3 100644 --- a/internal/core/src/index/knowhere/utils/Status.h +++ b/internal/core/src/index/knowhere/utils/Status.h @@ -9,71 +9,4 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. -#pragma once - -#include "utils/Error.h" - -#include - -namespace milvus { - -class Status; -#define STATUS_CHECK(func) \ - do { \ - Status s = func; \ - if (!s.ok()) { \ - return s; \ - } \ - } while (false) - -using StatusCode = ErrorCode; - -class Status { - public: - Status(StatusCode code, const std::string& msg); - Status() = default; - virtual ~Status(); - - Status(const Status& s); - - Status(Status&& s) noexcept; - - Status& - operator=(const Status& s); - - Status& - operator=(Status&& s) noexcept; - - static Status - OK() { - return Status(); - } - - bool - ok() const { - return state_.empty() || code() == 0; - } - - StatusCode - code() const { - return (state_.empty()) ? 0 : *(StatusCode*)(state_.data()); - } - - std::string - message() const; - - std::string - ToString() const; - - private: - inline void - CopyFrom(const Status& s); - - inline void - MoveFrom(Status& s); - - private: - std::string state_; -}; // Status - -} // namespace milvus +#include "../../../utils/Status.h" diff --git a/internal/core/src/index/knowhere/utils/StringHelpFunctions.cpp b/internal/core/src/index/knowhere/utils/StringHelpFunctions.cpp deleted file mode 100644 index c2da097037..0000000000 --- a/internal/core/src/index/knowhere/utils/StringHelpFunctions.cpp +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "utils/StringHelpFunctions.h" -#include "utils/Log.h" - -#include -#include -#include -#include - -namespace milvus { - -void -StringHelpFunctions::TrimStringBlank(std::string& string) { - if (!string.empty()) { - static std::string s_format(" \n\r\t"); - string.erase(0, string.find_first_not_of(s_format)); - string.erase(string.find_last_not_of(s_format) + 1); - } -} - -void -StringHelpFunctions::TrimStringQuote(std::string& string, const std::string& qoute) { - if (!string.empty()) { - string.erase(0, string.find_first_not_of(qoute)); - string.erase(string.find_last_not_of(qoute) + 1); - } -} - -void -StringHelpFunctions::SplitStringByDelimeter(const std::string& str, - const std::string& delimeter, - std::vector& result) { - if (str.empty()) { - return; - } - - size_t prev = 0; - while (true) { - size_t pos = str.find_first_of(delimeter, prev); - if (pos == std::string::npos) { - result.emplace_back(str.substr(prev)); - break; - } else { - result.emplace_back(str.substr(prev, pos - prev)); - prev = pos + 1; - } - } -} - -void -StringHelpFunctions::MergeStringWithDelimeter(const std::vector& strs, - const std::string& delimeter, - std::string& result) { - if (strs.empty()) { - result = ""; - return; - } - - result = strs[0]; - for (size_t i = 1; i < strs.size(); i++) { - result = result + delimeter + strs[i]; - } -} - -Status -StringHelpFunctions::SplitStringByQuote(const std::string& str, - const std::string& delimeter, - const std::string& quote, - std::vector& result) { - if (quote.empty()) { - SplitStringByDelimeter(str, delimeter, result); - return Status::OK(); - } - - size_t last = 0; - size_t index = str.find_first_of(quote, last); - if (index == std::string::npos) { - SplitStringByDelimeter(str, delimeter, result); - return Status::OK(); - } - - std::string process_str = str; - while (index != std::string::npos) { - std::string prefix = process_str.substr(last, index - last); - std::string append_prefix; - if (!prefix.empty()) { - std::vector prefix_split; - SplitStringByDelimeter(prefix, delimeter, prefix_split); - for (size_t i = 0; i < prefix_split.size() - 1; i++) { - result.push_back(prefix_split[i]); - } - append_prefix = prefix_split[prefix_split.size() - 1]; - } - last = index + 1; - std::string postfix = process_str.substr(last); - index = postfix.find_first_of(quote, 0); - fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index", index = std::string::npos); - - if (index == std::string::npos) { - return Status(SERVER_UNEXPECTED_ERROR, ""); - } - std::string quoted_text = postfix.substr(0, index); - append_prefix += quoted_text; - - last = index + 1; - index = postfix.find_first_of(delimeter, last); - fiu_do_on("StringHelpFunctions.SplitStringByQuote.index_gt_last", last = 0); - fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index2", index = std::string::npos); - - if (index != std::string::npos) { - if (index > last) { - append_prefix += postfix.substr(last, index - last); - } - } else { - append_prefix += postfix.substr(last); - } - result.emplace_back(append_prefix); - fiu_do_on("StringHelpFunctions.SplitStringByQuote.last_is_end", last = postfix.length()); - - if (last == postfix.length()) { - return Status::OK(); - } - - process_str = postfix.substr(index + 1); - last = 0; - index = process_str.find_first_of(quote, last); - } - - if (!process_str.empty()) { - SplitStringByDelimeter(process_str, delimeter, result); - } - - return Status::OK(); -} - -bool -StringHelpFunctions::IsRegexMatch(const std::string& target_str, const std::string& pattern_str) { - // if target_str equals pattern_str, return true - if (target_str == pattern_str) { - return true; - } - - // regex match - // for illegal regex expression, the std::regex will throw exception, regard as unmatch - try { - std::regex pattern(pattern_str); - std::smatch results; - return std::regex_match(target_str, results, pattern); - } catch (std::exception& e) { - LOG_SERVER_ERROR_ << "Regex exception: " << e.what(); - } - - return false; -} - -Status -StringHelpFunctions::ConvertToBoolean(const std::string& str, bool& value) { - std::string s = str; - std::transform(s.begin(), s.end(), s.begin(), ::tolower); - value = s == "true" || s == "on" || s == "yes" || s == "1"; - - return Status::OK(); -} - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/StringHelpFunctions.h b/internal/core/src/index/knowhere/utils/StringHelpFunctions.h deleted file mode 100644 index bcb5c38052..0000000000 --- a/internal/core/src/index/knowhere/utils/StringHelpFunctions.h +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include "utils/Status.h" - -#include -#include - -namespace milvus { - -class StringHelpFunctions { - private: - StringHelpFunctions() = default; - - public: - // trim blanks from begin and end - // " a b c " => "a b c" - static void - TrimStringBlank(std::string& string); - - // trim quotes from begin and end - // "'abc'" => "abc" - static void - TrimStringQuote(std::string& string, const std::string& qoute); - - // split string by delimeter ',' - // a,b,c a | b | c - // a,b, a | b | - // ,b,c | b | c - // ,b, | b | - // ,, | | - // a a - static void - SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector& result); - - // merge strings with delimeter - // "a", "b", "c" => "a,b,c" - static void - MergeStringWithDelimeter(const std::vector& strs, const std::string& delimeter, std::string& result); - - // assume the collection has two columns, quote='\"', delimeter=',' - // a,b a | b - // "aa,gg,yy",b aa,gg,yy | b - // aa"dd,rr"kk,pp aadd,rrkk | pp - // "aa,bb" aa,bb - // 55,1122\"aa,bb\",yyy,\"kkk\" 55 | 1122aa,bb | yyy | kkk - // "55,1122"aa,bb",yyy,"kkk" illegal - static Status - SplitStringByQuote(const std::string& str, - const std::string& delimeter, - const std::string& quote, - std::vector& result); - - // std regex match function - // regex grammar reference: http://www.cplusplus.com/reference/regex/ECMAScript/ - static bool - IsRegexMatch(const std::string& target_str, const std::string& pattern); - - // conversion rules refer to ValidationUtil::ValidateStringIsBool() - // "true", "on", "yes", "1" ==> true - // "false", "off", "no", "0", "" ==> false - static Status - ConvertToBoolean(const std::string& str, bool& value); -}; - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/ThreadPool.h b/internal/core/src/index/knowhere/utils/ThreadPool.h deleted file mode 100644 index 1e6f0f966c..0000000000 --- a/internal/core/src/index/knowhere/utils/ThreadPool.h +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define MAX_THREADS_NUM 32 - -namespace milvus { - -class ThreadPool { - public: - explicit ThreadPool(size_t threads, size_t queue_size = 1000); - - template - auto - enqueue(F&& f, Args&&... args) -> std::future::type>; - - void - Stop(); - - ~ThreadPool(); - - private: - // need to keep track of threads so we can join them - std::vector workers_; - - // the task queue - std::queue> tasks_; - - size_t max_queue_size_; - - // synchronization - std::mutex queue_mutex_; - - std::condition_variable condition_; - - std::atomic_bool stop_; -}; - -// the constructor just launches some amount of workers -inline ThreadPool::ThreadPool(size_t threads, size_t queue_size) : max_queue_size_(queue_size), stop_(false) { - for (size_t i = 0; i < threads; ++i) - workers_.emplace_back([this] { - for (;;) { - std::function task; - - { - std::unique_lock lock(this->queue_mutex_); - this->condition_.wait(lock, [this] { return this->stop_ || !this->tasks_.empty(); }); - if (this->stop_ && this->tasks_.empty()) - return; - task = std::move(this->tasks_.front()); - this->tasks_.pop(); - } - this->condition_.notify_all(); - - task(); - } - }); -} - -// add new work item to the pool -template -auto -ThreadPool::enqueue(F&& f, Args&&... args) -> std::future::type> { - using return_type = typename std::result_of::type; - - auto task = - std::make_shared>(std::bind(std::forward(f), std::forward(args)...)); - fiu_do_on("ThreadPool.enqueue.stop_is_true", stop_ = true); - std::future res = task->get_future(); - { - std::unique_lock lock(queue_mutex_); - this->condition_.wait(lock, [this] { return this->tasks_.size() < max_queue_size_; }); - // don't allow enqueueing after stopping the pool - if (stop_) - throw std::runtime_error("enqueue on stopped ThreadPool"); - - tasks_.emplace([task]() { (*task)(); }); - } - condition_.notify_all(); - return res; -} - -inline void -ThreadPool::Stop() { - if (stop_) { - return; - } - { - std::unique_lock lock(queue_mutex_); - stop_ = true; - } - condition_.notify_all(); - for (std::thread& worker : workers_) { - worker.join(); - } -} - -// the destructor joins all threads -inline ThreadPool::~ThreadPool() { - Stop(); -} - -using ThreadPoolPtr = std::shared_ptr; - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/TimeRecorder.cpp b/internal/core/src/index/knowhere/utils/TimeRecorder.cpp deleted file mode 100644 index f4d6bff80b..0000000000 --- a/internal/core/src/index/knowhere/utils/TimeRecorder.cpp +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include - -#include "utils/Log.h" -#include "utils/TimeRecorder.h" - -namespace milvus { - -TimeRecorder::TimeRecorder(std::string hdr, int64_t log_level) : header_(std::move(hdr)), log_level_(log_level) { - start_ = last_ = stdclock::now(); -} - -std::string -TimeRecorder::GetTimeSpanStr(double span) { - std::string str_sec = std::to_string(span * 0.000001) + ((span > 1000000) ? " seconds" : " second"); - std::string str_ms = std::to_string(span * 0.001) + " ms"; - - return str_sec + " [" + str_ms + "]"; -} - -void -TimeRecorder::PrintTimeRecord(const std::string& msg, double span) { - std::string str_log; - if (!header_.empty()) { - str_log += header_ + ": "; - } - str_log += msg; - str_log += " ("; - str_log += TimeRecorder::GetTimeSpanStr(span); - str_log += ")"; - - switch (log_level_) { - case 0: - LOG_SERVER_TRACE_ << str_log; - break; - case 1: - LOG_SERVER_DEBUG_ << str_log; - break; - case 2: - LOG_SERVER_INFO_ << str_log; - break; - case 3: - LOG_SERVER_WARNING_ << str_log; - break; - case 4: - LOG_SERVER_ERROR_ << str_log; - break; - case 5: - LOG_SERVER_FATAL_ << str_log; - break; - default: - LOG_SERVER_INFO_ << str_log; - break; - } -} - -double -TimeRecorder::RecordSection(const std::string& msg) { - stdclock::time_point curr = stdclock::now(); - double span = (std::chrono::duration(curr - last_)).count(); - last_ = curr; - - PrintTimeRecord(msg, span); - return span; -} - -double -TimeRecorder::ElapseFromBegin(const std::string& msg) { - stdclock::time_point curr = stdclock::now(); - double span = (std::chrono::duration(curr - start_)).count(); - - PrintTimeRecord(msg, span); - return span; -} - -TimeRecorderAuto::TimeRecorderAuto(std::string hdr, int64_t log_level) : TimeRecorder(hdr, log_level) { -} - -TimeRecorderAuto::~TimeRecorderAuto() { - ElapseFromBegin("totally cost"); -} - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/TimeRecorder.h b/internal/core/src/index/knowhere/utils/TimeRecorder.h deleted file mode 100644 index ffab6c5b88..0000000000 --- a/internal/core/src/index/knowhere/utils/TimeRecorder.h +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include "utils/Log.h" - -namespace milvus { - -inline void -print_timestamp(const std::string& message) { - std::chrono::time_point now = std::chrono::system_clock::now(); - auto duration = now.time_since_epoch(); - auto micros = std::chrono::duration_cast(duration).count(); - micros %= 1000000; - double millisecond = (double)micros / 1000.0; - - LOG_SERVER_DEBUG_ << std::fixed << " " << millisecond << "(ms) [timestamp]" << message; -} - -class TimeRecorder { - using stdclock = std::chrono::high_resolution_clock; - - public: - // trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5 - explicit TimeRecorder(std::string hdr, int64_t log_level = 1); - virtual ~TimeRecorder() = default; - - double - RecordSection(const std::string& msg); - - double - ElapseFromBegin(const std::string& msg); - - static std::string - GetTimeSpanStr(double span); - - private: - void - PrintTimeRecord(const std::string& msg, double span); - - private: - std::string header_; - stdclock::time_point start_; - stdclock::time_point last_; - int64_t log_level_; -}; - -class TimeRecorderAuto : public TimeRecorder { - public: - explicit TimeRecorderAuto(std::string hdr, int64_t log_level = 1); - ~TimeRecorderAuto() override; -}; - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/TimerContext.h b/internal/core/src/index/knowhere/utils/TimerContext.h deleted file mode 100644 index 0396c4ef61..0000000000 --- a/internal/core/src/index/knowhere/utils/TimerContext.h +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "utils/Log.h" -#include "utils/ThreadPool.h" - -namespace milvus { - -struct TimerContext { - using HandlerT = std::function; - struct Context { - /* Context(int interval_us, HandlerT& handler, ThreadPoolPtr pool = nullptr) */ - /* : interval_(interval_us), handler_(handler), timer_(io, interval_), pool_(pool) { */ - /* } */ - int interval_us; - HandlerT handler; - ThreadPoolPtr pool = nullptr; - }; - - TimerContext(boost::asio::io_service& io, int interval_us, HandlerT& handler, ThreadPoolPtr pool) - : io_(io), interval_(interval_us), handler_(handler), timer_(io, interval_), pool_(pool) { - } - TimerContext(boost::asio::io_service& io, Context& context) - : io_(io), - interval_(context.interval_us), - handler_(context.handler), - timer_(io, interval_), - pool_(context.pool) { - } - - void - Reschedule(const boost::system::error_code& ec); - - boost::asio::io_service& io_; - boost::posix_time::microseconds interval_; - boost::asio::deadline_timer timer_; - HandlerT handler_; - ThreadPoolPtr pool_; -}; - -inline void -TimerContext::Reschedule(const boost::system::error_code& ec) { - try { - pool_->enqueue(handler_, ec); - } catch (std::exception& ex) { - LOG_SERVER_ERROR_ << "Fail to enqueue handler: " << std::string(ex.what()); - } - boost::system::error_code e; - auto new_expires = timer_.expires_at() + interval_; - timer_.expires_at(new_expires, e); - if (e) { - LOG_SERVER_ERROR_ << "Fail to Reschedule: " << e; - } - timer_.async_wait(std::bind(&TimerContext::Reschedule, this, std::placeholders::_1)); -} - -using TimerContextPtr = std::shared_ptr; - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/TimerManager.cpp b/internal/core/src/index/knowhere/utils/TimerManager.cpp deleted file mode 100644 index a52893c86f..0000000000 --- a/internal/core/src/index/knowhere/utils/TimerManager.cpp +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include - -#include "utils/Log.h" -#include "utils/TimerManager.h" - -namespace milvus { - -TimerManager::TimerManager(unsigned int pool_size) : pool_size_(pool_size) { -} - -TimerManager::~TimerManager() { -} - -Status -TimerManager::SetPoolSize(unsigned int pool_size) { - if (timer_exeutors_) { - return Status(SERVER_UNEXPECTED_ERROR, "Cannot set pool size since it has been set already"); - } - pool_size_ = pool_size; - return Status::OK(); -} - -Status -TimerManager::Run() { - boost::system::error_code ec; - io_.run(ec); - Status status; - if (ec) { - status = Status(SERVER_UNEXPECTED_ERROR, ec.message()); - } - return status; -} - -Status -TimerManager::Start() { - for (auto& timer : timers_) { - timer->timer_.async_wait(std::bind(&TimerContext::Reschedule, timer, std::placeholders::_1)); - } - return Status::OK(); -} - -void -TimerManager::Stop() { - boost::system::error_code ec; - for (auto& timer : timers_) { - timer->timer_.cancel(ec); - if (ec) { - LOG_SERVER_ERROR_ << "Fail to cancel timer: " << ec; - } - } - if (timer_exeutors_) { - timer_exeutors_->Stop(); - } -} - -void -TimerManager::AddTimer(int interval_us, TimerContext::HandlerT handler) { - if (!timer_exeutors_) { - timer_exeutors_ = std::make_shared(pool_size_); - } - timers_.emplace_back(std::make_shared(io_, interval_us, handler, timer_exeutors_)); -} - -void -TimerManager::AddTimer(const TimerContext::Context& ctx) { - if (!timer_exeutors_) { - timer_exeutors_ = std::make_shared(pool_size_); - } - TimerContext::Context context(ctx); - context.pool = timer_exeutors_; - timers_.emplace_back(std::make_shared(io_, context)); -} - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/TimerManager.h b/internal/core/src/index/knowhere/utils/TimerManager.h deleted file mode 100644 index 5e5441faae..0000000000 --- a/internal/core/src/index/knowhere/utils/TimerManager.h +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include -#include - -#include "utils/Status.h" -#include "utils/ThreadPool.h" -#include "utils/TimerContext.h" - -namespace milvus { - -class TimerManager { - public: - TimerManager() = default; - explicit TimerManager(unsigned int pool_size); - - Status - SetPoolSize(unsigned int pool_size); - - void - AddTimer(int interval_us, TimerContext::HandlerT handler); - void - AddTimer(const TimerContext::Context& ctx); - - virtual Status - Run(); - - virtual Status - Start(); - virtual void - Stop(); - - virtual ~TimerManager(); - - protected: - boost::asio::io_service io_; - ThreadPoolPtr timer_exeutors_; - unsigned int pool_size_; - std::vector timers_; -}; - -using TimerManagerPtr = std::shared_ptr; - -} // namespace milvus diff --git a/internal/core/src/index/knowhere/utils/Types.cpp b/internal/core/src/index/knowhere/utils/Types.cpp deleted file mode 100644 index ab50d1ec72..0000000000 --- a/internal/core/src/index/knowhere/utils/Types.cpp +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License - -#include "Types.h" diff --git a/internal/core/src/index/knowhere/utils/Types.h b/internal/core/src/index/knowhere/utils/Types.h deleted file mode 100644 index 4dff86801e..0000000000 --- a/internal/core/src/index/knowhere/utils/Types.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -// #include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace milvus::engine { - -using idx_t = int64_t; -using offset_t = int32_t; -using date_t = int32_t; -using distance_t = float; - -using IDNumbers = std::vector; - -enum class DataType { - NONE = 0, - BOOL = 1, - INT8 = 2, - INT16 = 3, - INT32 = 4, - INT64 = 5, - - FLOAT = 10, - DOUBLE = 11, - - STRING = 20, - - VECTOR_BINARY = 100, - VECTOR_FLOAT = 101, -}; - -} // namespace milvus::engine diff --git a/internal/core/src/index/knowhere/utils/tools.h b/internal/core/src/index/knowhere/utils/tools.h deleted file mode 100644 index b99ddda6ef..0000000000 --- a/internal/core/src/index/knowhere/utils/tools.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License - -#pragma once -#include -#include "exceptions/EasyAssert.h" -namespace milvus { -inline int64_t -upper_align(int64_t value, int64_t align) { - Assert(align > 0); - auto groups = (value + align - 1) / align; - return groups * align; -} - -inline int64_t -upper_div(int64_t value, int64_t align) { - Assert(align > 0); - auto groups = (value + align - 1) / align; - return groups; -} - -} // namespace milvus diff --git a/internal/core/src/utils/BlockingQueue.h b/internal/core/src/utils/BlockingQueue.h deleted file mode 100644 index 1b489f53e6..0000000000 --- a/internal/core/src/utils/BlockingQueue.h +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include - -namespace milvus { - -template -class BlockingQueue { - public: - BlockingQueue() : mtx(), full_(), empty_() { - } - - virtual ~BlockingQueue() { - } - - BlockingQueue(const BlockingQueue& rhs) = delete; - - BlockingQueue& - operator=(const BlockingQueue& rhs) = delete; - - void - Put(const T& task) { - std::unique_lock lock(mtx); - full_.wait(lock, [this] { return (queue_.size() < capacity_); }); - queue_.push(task); - empty_.notify_all(); - } - - T - Take() { - std::unique_lock lock(mtx); - empty_.wait(lock, [this] { return !queue_.empty(); }); - T front(queue_.front()); - queue_.pop(); - full_.notify_all(); - return front; - } - - T - Front() { - std::unique_lock lock(mtx); - empty_.wait(lock, [this] { return !queue_.empty(); }); - T front(queue_.front()); - return front; - } - - T - Back() { - std::unique_lock lock(mtx); - empty_.wait(lock, [this] { return !queue_.empty(); }); - T back(queue_.back()); - return back; - } - - size_t - Size() const { - std::lock_guard lock(mtx); - return queue_.size(); - } - - bool - Empty() const { - std::unique_lock lock(mtx); - return queue_.empty(); - } - - void - SetCapacity(const size_t capacity) { - capacity_ = (capacity > 0 ? capacity : capacity_); - } - - protected: - mutable std::mutex mtx; - std::condition_variable full_; - std::condition_variable empty_; - std::queue queue_; - size_t capacity_ = 32; -}; - -} // namespace milvus diff --git a/internal/core/src/utils/CommonUtil.cpp b/internal/core/src/utils/CommonUtil.cpp deleted file mode 100644 index f2b4215972..0000000000 --- a/internal/core/src/utils/CommonUtil.cpp +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "utils/CommonUtil.h" -#include "utils/Log.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace milvus { - -namespace fs = boost::filesystem; - -bool -CommonUtil::IsDirectoryExist(const std::string& path) { - DIR* dp = nullptr; - if ((dp = opendir(path.c_str())) == nullptr) { - return false; - } - - closedir(dp); - return true; -} - -Status -CommonUtil::CreateDirectory(const std::string& path) { - if (path.empty()) { - return Status::OK(); - } - - struct stat directory_stat; - int status = stat(path.c_str(), &directory_stat); - if (status == 0) { - return Status::OK(); // already exist - } - - fs::path fs_path(path); - fs::path parent_path = fs_path.parent_path(); - Status err_status = CreateDirectory(parent_path.string()); - fiu_do_on("CommonUtil.CreateDirectory.create_parent_fail", err_status = Status(SERVER_INVALID_ARGUMENT, "")); - if (!err_status.ok()) { - return err_status; - } - - status = stat(path.c_str(), &directory_stat); - if (status == 0) { - return Status::OK(); // already exist - } - - int makeOK = mkdir(path.c_str(), S_IRWXU | S_IRGRP | S_IROTH); - fiu_do_on("CommonUtil.CreateDirectory.create_dir_fail", makeOK = 1); - if (makeOK != 0) { - return Status(SERVER_UNEXPECTED_ERROR, "failed to create directory: " + path); - } - - return Status::OK(); -} - -namespace { -void -RemoveDirectory(const std::string& path) { - DIR* dir = nullptr; - const int32_t buf_size = 256; - char file_name[buf_size]; - - std::string folder_name = path + "/%s"; - if ((dir = opendir(path.c_str())) != nullptr) { - struct dirent* dmsg; - while ((dmsg = readdir(dir)) != nullptr) { - if (strcmp(dmsg->d_name, ".") != 0 && strcmp(dmsg->d_name, "..") != 0) { - snprintf(file_name, buf_size, folder_name.c_str(), dmsg->d_name); - std::string tmp = file_name; - if (tmp.find('.') == std::string::npos) { - RemoveDirectory(file_name); - } - remove(file_name); - } - } - } - - if (dir != nullptr) { - closedir(dir); - } - remove(path.c_str()); -} -} // namespace - -Status -CommonUtil::DeleteDirectory(const std::string& path) { - if (path.empty()) { - return Status::OK(); - } - - struct stat directory_stat; - int statOK = stat(path.c_str(), &directory_stat); - if (statOK != 0) { - return Status::OK(); - } - - RemoveDirectory(path); - return Status::OK(); -} - -bool -CommonUtil::IsFileExist(const std::string& path) { - return (access(path.c_str(), F_OK) == 0); -} - -uint64_t -CommonUtil::GetFileSize(const std::string& path) { - struct stat file_info; - if (stat(path.c_str(), &file_info) < 0) { - return 0; - } - - return static_cast(file_info.st_size); -} - -std::string -CommonUtil::GetFileName(std::string filename) { - int pos = filename.find_last_of('/'); - return filename.substr(pos + 1); -} - -std::string -CommonUtil::GetExePath() { - const int64_t buf_len = 1024; - char buf[buf_len]; - int64_t cnt = readlink("/proc/self/exe", buf, buf_len); - fiu_do_on("CommonUtil.GetExePath.readlink_fail", cnt = -1); - if (cnt < 0 || cnt >= buf_len) { - return ""; - } - - buf[cnt] = '\0'; - - std::string exe_path = buf; - fiu_do_on("CommonUtil.GetExePath.exe_path_error", exe_path = "/"); - if (exe_path.rfind('/') != exe_path.length() - 1) { - std::string sub_str = exe_path.substr(0, exe_path.rfind('/')); - return sub_str + "/"; - } - return exe_path; -} - -bool -CommonUtil::TimeStrToTime(const std::string& time_str, - time_t& time_integer, - tm& time_struct, - const std::string& format) { - time_integer = 0; - memset(&time_struct, 0, sizeof(tm)); - - int ret = sscanf(time_str.c_str(), format.c_str(), &(time_struct.tm_year), &(time_struct.tm_mon), - &(time_struct.tm_mday), &(time_struct.tm_hour), &(time_struct.tm_min), &(time_struct.tm_sec)); - if (ret <= 0) { - return false; - } - - time_struct.tm_year -= 1900; - time_struct.tm_mon--; - time_integer = mktime(&time_struct); - - return true; -} - -void -CommonUtil::GetCurrentTimeStr(std::string& time_str) { - auto t = std::time(nullptr); - struct tm ltm; - localtime_r(&t, <m); - - time_str = ""; - time_str += std::to_string(ltm.tm_year + 1900); - time_str += "-"; - time_str += std::to_string(ltm.tm_mon + 1); - time_str += "-"; - time_str += std::to_string(ltm.tm_mday); - time_str += "_"; - time_str += std::to_string(ltm.tm_hour); - time_str += ":"; - time_str += std::to_string(ltm.tm_min); - time_str += ":"; - time_str += std::to_string(ltm.tm_sec); -} - -void -CommonUtil::ConvertTime(time_t time_integer, tm& time_struct) { - localtime_r(&time_integer, &time_struct); -} - -void -CommonUtil::ConvertTime(tm time_struct, time_t& time_integer) { - time_integer = mktime(&time_struct); -} - -std::string -CommonUtil::ConvertSize(int64_t size) { - const int64_t gb = 1024ll * 1024 * 1024; - const int64_t mb = 1024ll * 1024; - const int64_t kb = 1024ll; - if (size % gb == 0) { - return std::to_string(size / gb) + "GB"; - } else if (size % mb == 0) { - return std::to_string(size / mb) + "MB"; - } else if (size % kb == 0) { - return std::to_string(size / kb) + "KB"; - } else { - return std::to_string(size); - } -} - -#ifdef ENABLE_CPU_PROFILING -std::string -CommonUtil::GetCurrentTimeStr() { - time_t tt; - time(&tt); - tt = tt + 8 * 60; - tm t; - gmtime_r(&tt, &t); - - std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) + "_" + - std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) + "_" + std::to_string(t.tm_min) + - "_" + std::to_string(t.tm_sec); - return str; -} -#endif - -} // namespace milvus diff --git a/internal/core/src/utils/CommonUtil.h b/internal/core/src/utils/CommonUtil.h deleted file mode 100644 index 4da0a30505..0000000000 --- a/internal/core/src/utils/CommonUtil.h +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include "utils/Status.h" - -#include -#include - -namespace milvus { - -class CommonUtil { - public: - static bool - IsFileExist(const std::string& path); - static uint64_t - GetFileSize(const std::string& path); - static bool - IsDirectoryExist(const std::string& path); - static Status - CreateDirectory(const std::string& path); - static Status - DeleteDirectory(const std::string& path); - - static std::string - GetFileName(std::string filename); - static std::string - GetExePath(); - - static bool - TimeStrToTime(const std::string& time_str, - time_t& time_integer, - tm& time_struct, - const std::string& format = "%d-%d-%d %d:%d:%d"); - - static void - GetCurrentTimeStr(std::string& time_str); - - static void - ConvertTime(time_t time_integer, tm& time_struct); - static void - ConvertTime(tm time_struct, time_t& time_integer); - - static std::string - ConvertSize(int64_t size); - -#ifdef ENABLE_CPU_PROFILING - static std::string - GetCurrentTimeStr(); -#endif -}; - -} // namespace milvus diff --git a/internal/core/src/utils/ConfigUtils.cpp b/internal/core/src/utils/ConfigUtils.cpp deleted file mode 100644 index 8f459954d4..0000000000 --- a/internal/core/src/utils/ConfigUtils.cpp +++ /dev/null @@ -1,320 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "utils/ConfigUtils.h" -#include "utils/Log.h" -#include "utils/StringHelpFunctions.h" - -#include -#include -#include -#ifdef MILVUS_GPU_VERSION -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(__x86_64__) -#define THREAD_MULTIPLY_CPU 1 -#elif defined(__powerpc64__) -#define THREAD_MULTIPLY_CPU 4 -#else -#define THREAD_MULTIPLY_CPU 1 -#endif - -namespace milvus { -namespace server { - -std::unordered_map BYTE_UNITS = { - {"b", 1}, - {"k", 1024}, - {"m", 1024 * 1024}, - {"g", 1024 * 1024 * 1024}, -}; - -bool -is_number(const std::string& s) { - return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isdigit(c); }) == s.end(); -} - -bool -is_alpha(const std::string& s) { - return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isalpha(c); }) == s.end(); -} - -std::string -str_tolower(std::string s) { - std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); }); - return s; -} - -int64_t -parse_bytes(const std::string& str, std::string& err) { - try { - std::string s = str; - if (is_number(s)) { - return std::stoll(s); - } - if (s.length() == 0) { - return 0; - } - - auto last_two = s.substr(s.length() - 2, 2); - auto last_one = s.substr(s.length() - 1); - if (is_alpha(last_two) && is_alpha(last_one)) { - if (last_one == "b" or last_one == "B") { - s = s.substr(0, s.length() - 1); - } - } - auto& units = BYTE_UNITS; - auto suffix = str_tolower(s.substr(s.length() - 1)); - - std::string digits_part; - if (is_number(suffix)) { - digits_part = s; - suffix = 'b'; - } else { - digits_part = s.substr(0, s.length() - 1); - } - - if (units.find(suffix) != units.end() or is_number(suffix)) { - auto digits = std::stoll(digits_part); - return digits * units[suffix]; - } else { - std::stringstream ss; - ss << "The specified value for memory (" << str << ") should specify the units." - << "The postfix should be one of the `b` `k` `m` `g` characters"; - err = ss.str(); - } - } catch (...) { - err = "Unknown error happened on parse bytes."; - } - return 0; -} - -bool -GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem) { - struct sysinfo info; - int ret = sysinfo(&info); - total_mem = info.totalram; - free_mem = info.freeram; - - return ret == 0; // succeed 0, failed -1 -} - -bool -GetSystemAvailableThreads(int64_t& thread_count) { - // threadCnt = std::thread::hardware_concurrency(); - thread_count = sysconf(_SC_NPROCESSORS_CONF); - thread_count *= THREAD_MULTIPLY_CPU; - fiu_do_on("GetSystemAvailableThreads.zero_thread", thread_count = 0); - - if (thread_count == 0) { - thread_count = 8; - } - - return true; -} - -Status -ValidateGpuIndex(int32_t gpu_index) { -#ifdef MILVUS_GPU_VERSION - int num_devices = 0; - auto cuda_err = cudaGetDeviceCount(&num_devices); - fiu_do_on("config.ValidateGpuIndex.get_device_count_fail", cuda_err = cudaError::cudaErrorUnknown); - - if (cuda_err != cudaSuccess) { - std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err); - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_UNEXPECTED_ERROR, msg); - } - - if (gpu_index >= num_devices) { - std::string msg = "Invalid gpu index: " + std::to_string(gpu_index); - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - } -#endif - - return Status::OK(); -} - -#ifdef MILVUS_GPU_VERSION -Status -GetGpuMemory(int32_t gpu_index, int64_t& memory) { - fiu_return_on("config.GetGpuMemory.return_error", Status(SERVER_UNEXPECTED_ERROR, "")); - - cudaDeviceProp deviceProp; - auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index); - if (cuda_err) { - std::string msg = "Failed to get gpu properties for gpu" + std::to_string(gpu_index) + - " , cuda error:" + std::to_string(cuda_err); - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_UNEXPECTED_ERROR, msg); - } - - memory = deviceProp.totalGlobalMem; - return Status::OK(); -} -#endif - -Status -ValidateIpAddress(const std::string& ip_address) { - struct in_addr address; - - int result = inet_pton(AF_INET, ip_address.c_str(), &address); - fiu_do_on("config.ValidateIpAddress.error_ip_result", result = 2); - - switch (result) { - case 1: - return Status::OK(); - case 0: { - std::string msg = "Invalid IP address: " + ip_address; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - default: { - std::string msg = "IP address conversion error: " + ip_address; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_UNEXPECTED_ERROR, msg); - } - } -} - -Status -ValidateStringIsNumber(const std::string& str) { - if (str.empty() || !std::all_of(str.begin(), str.end(), ::isdigit)) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid number"); - } - try { - int64_t value = std::stol(str); - fiu_do_on("config.ValidateStringIsNumber.throw_exception", throw std::exception()); - if (value < 0) { - return Status(SERVER_INVALID_ARGUMENT, "Negative number"); - } - } catch (...) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid number"); - } - return Status::OK(); -} - -Status -ValidateStringIsBool(const std::string& str) { - fiu_return_on("ValidateStringNotBool", Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str)); - std::string s = str; - std::transform(s.begin(), s.end(), s.begin(), ::tolower); - if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" || - s.empty()) { - return Status::OK(); - } - return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str); -} - -Status -ValidateStringIsFloat(const std::string& str) { - try { - float val = std::stof(str); - if (val < 0.0) { - return Status(SERVER_INVALID_ARGUMENT, "Negative float: " + str); - } - } catch (...) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str); - } - return Status::OK(); -} - -Status -ValidateDbURI(const std::string& uri) { - std::string dialectRegex = "(.*)"; - std::string usernameRegex = "(.*)"; - std::string passwordRegex = "(.*)"; - std::string hostRegex = "(.*)"; - std::string portRegex = "(.*)"; - std::string dbNameRegex = "(.*)"; - std::string uriRegexStr = dialectRegex + R"(\:\/\/)" + usernameRegex + R"(\:)" + passwordRegex + R"(\@)" + - hostRegex + R"(\:)" + portRegex + R"(\/)" + dbNameRegex; - std::regex uriRegex(uriRegexStr); - std::smatch pieces_match; - - bool okay = true; - - if (std::regex_match(uri, pieces_match, uriRegex)) { - std::string dialect = pieces_match[1].str(); - std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); - if (dialect.find("mysql") == std::string::npos && dialect.find("sqlite") == std::string::npos && - dialect.find("mock") == std::string::npos) { - LOG_SERVER_ERROR_ << "Invalid dialect in URI: dialect = " << dialect; - okay = false; - } - - /* - * Could be DNS, skip checking - * - std::string host = pieces_match[4].str(); - if (!host.empty() && host != "localhost") { - if (ValidateIpAddress(host) != SERVER_SUCCESS) { - LOG_SERVER_ERROR_ << "Invalid host ip address in uri = " << host; - okay = false; - } - } - */ - - std::string port = pieces_match[5].str(); - if (!port.empty()) { - auto status = ValidateStringIsNumber(port); - if (!status.ok()) { - LOG_SERVER_ERROR_ << "Invalid port in uri = " << port; - okay = false; - } - } - } else { - LOG_SERVER_ERROR_ << "Wrong URI format: URI = " << uri; - okay = false; - } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid db backend uri")); -} - -Status -ValidateStoragePath(const std::string& path) { - // Validate storage path if is valid, only correct absolute path will be validated pass - // Invalid path only contain character[a-zA-Z], number[0-9], '-', and '_', - // and path must start with '/'. - // examples below are invalid - // '/a//a', '/a--/a', '/-a/a', '/a@#/a', 'aaa/sfs' - std::string path_pattern = R"(^\/(\w+-?\/?)+$)"; - std::regex regex(path_pattern); - - return std::regex_match(path, regex) ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid file path"); -} - -Status -ValidateLogLevel(const std::string& level) { - std::set supported_level{"debug", "info", "warning", "error", "fatal"}; - - return supported_level.find(level) != supported_level.end() - ? Status::OK() - : Status(SERVER_INVALID_ARGUMENT, "Log level must be one of debug, info, warning, error and fatal."); -} - -bool -IsNumber(const std::string& s) { - return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit); -} - -} // namespace server -} // namespace milvus diff --git a/internal/core/src/utils/ConfigUtils.h b/internal/core/src/utils/ConfigUtils.h deleted file mode 100644 index 27d3ce6842..0000000000 --- a/internal/core/src/utils/ConfigUtils.h +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include - -#include "utils/Status.h" - -namespace milvus { -namespace server { - -extern int64_t -parse_bytes(const std::string& str, std::string& err); - -extern bool -GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem); - -extern bool -GetSystemAvailableThreads(int64_t& thread_count); - -extern Status -ValidateGpuIndex(int32_t gpu_index); - -#ifdef MILVUS_GPU_VERSION -extern Status -GetGpuMemory(int32_t gpu_index, int64_t& memory); -#endif - -extern Status -ValidateIpAddress(const std::string& ip_address); - -extern Status -ValidateStringIsNumber(const std::string& str); - -extern Status -ValidateStringIsBool(const std::string& str); - -extern Status -ValidateStringIsFloat(const std::string& str); - -extern Status -ValidateDbURI(const std::string& uri); - -extern Status -ValidateStoragePath(const std::string& path); - -extern Status -ValidateLogLevel(const std::string& level); - -extern bool -IsNumber(const std::string& s); -} // namespace server -} // namespace milvus diff --git a/internal/core/src/utils/Exception.h b/internal/core/src/utils/Exception.h deleted file mode 100644 index 454fcb508f..0000000000 --- a/internal/core/src/utils/Exception.h +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include "utils/Error.h" - -#include -#include -#include - -namespace milvus { - -#define THROW_ERROR(err_code, err_msg) \ - LOG_ENGINE_ERROR_ << err_msg; \ - throw Exception(err_code, err_msg); - -class Exception : public std::exception { - public: - Exception(ErrorCode code, std::string msg) : code_(code), message_(std::move(msg)) { - } - - ErrorCode - code() const noexcept { - return code_; - } - - const char* - what() const noexcept override { - if (message_.empty()) { - return "Default Exception."; - } else { - return message_.c_str(); - } - } - - ~Exception() noexcept override = default; - - protected: - ErrorCode code_; - std::string message_; -}; - -class InvalidArgumentException : public Exception { - public: - InvalidArgumentException() : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") { - } - - explicit InvalidArgumentException(const std::string& message) : Exception(SERVER_INVALID_ARGUMENT, message) { - } -}; - -} // namespace milvus diff --git a/internal/core/src/utils/SignalHandler.cpp b/internal/core/src/utils/SignalHandler.cpp deleted file mode 100644 index de234d795b..0000000000 --- a/internal/core/src/utils/SignalHandler.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "utils/SignalHandler.h" -#include "utils/Log.h" - -#include -#include - -namespace milvus { - -signal_func_ptr signal_routine_func = nullptr; - -void -HandleSignal(int signum) { - int32_t exit_code = 1; /* 0: normal exit; 1: exception */ - switch (signum) { - case SIGINT: - case SIGUSR2: - exit_code = 0; - /* no break */ - default: { - if (exit_code == 0) { - LOG_SERVER_INFO_ << "Server received signal: " << signum; - } else { - LOG_SERVER_INFO_ << "Server received critical signal: " << signum; - PrintStacktrace(); - } - if (signal_routine_func != nullptr) { - (*signal_routine_func)(exit_code); - } - } - } -} - -void -PrintStacktrace() { - const int bt_depth = 128; - void* array[bt_depth]; - int stack_num = backtrace(array, bt_depth); - char** stacktrace = backtrace_symbols(array, stack_num); - - LOG_SERVER_INFO_ << "Call stack:"; - for (int i = 0; i < stack_num; ++i) { - std::string info = stacktrace[i]; - std::cout << "No." << i << ": " << info << std::endl; - LOG_SERVER_INFO_ << info; - } - free(stacktrace); -} - -} // namespace milvus diff --git a/internal/core/src/utils/SignalHandler.h b/internal/core/src/utils/SignalHandler.h deleted file mode 100644 index 5c58886e45..0000000000 --- a/internal/core/src/utils/SignalHandler.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include - -namespace milvus { - -typedef void (*signal_func_ptr)(int32_t); - -extern signal_func_ptr signal_routine_func; - -extern void -HandleSignal(int signum); - -extern void -PrintStacktrace(); - -} // namespace milvus diff --git a/internal/core/src/utils/Status.h b/internal/core/src/utils/Status.h index 68342a3070..56948aea1a 100644 --- a/internal/core/src/utils/Status.h +++ b/internal/core/src/utils/Status.h @@ -11,7 +11,7 @@ #pragma once -#include "utils/Error.h" +#include "Error.h" #include diff --git a/internal/core/src/utils/StringHelpFunctions.cpp b/internal/core/src/utils/StringHelpFunctions.cpp deleted file mode 100644 index c2da097037..0000000000 --- a/internal/core/src/utils/StringHelpFunctions.cpp +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "utils/StringHelpFunctions.h" -#include "utils/Log.h" - -#include -#include -#include -#include - -namespace milvus { - -void -StringHelpFunctions::TrimStringBlank(std::string& string) { - if (!string.empty()) { - static std::string s_format(" \n\r\t"); - string.erase(0, string.find_first_not_of(s_format)); - string.erase(string.find_last_not_of(s_format) + 1); - } -} - -void -StringHelpFunctions::TrimStringQuote(std::string& string, const std::string& qoute) { - if (!string.empty()) { - string.erase(0, string.find_first_not_of(qoute)); - string.erase(string.find_last_not_of(qoute) + 1); - } -} - -void -StringHelpFunctions::SplitStringByDelimeter(const std::string& str, - const std::string& delimeter, - std::vector& result) { - if (str.empty()) { - return; - } - - size_t prev = 0; - while (true) { - size_t pos = str.find_first_of(delimeter, prev); - if (pos == std::string::npos) { - result.emplace_back(str.substr(prev)); - break; - } else { - result.emplace_back(str.substr(prev, pos - prev)); - prev = pos + 1; - } - } -} - -void -StringHelpFunctions::MergeStringWithDelimeter(const std::vector& strs, - const std::string& delimeter, - std::string& result) { - if (strs.empty()) { - result = ""; - return; - } - - result = strs[0]; - for (size_t i = 1; i < strs.size(); i++) { - result = result + delimeter + strs[i]; - } -} - -Status -StringHelpFunctions::SplitStringByQuote(const std::string& str, - const std::string& delimeter, - const std::string& quote, - std::vector& result) { - if (quote.empty()) { - SplitStringByDelimeter(str, delimeter, result); - return Status::OK(); - } - - size_t last = 0; - size_t index = str.find_first_of(quote, last); - if (index == std::string::npos) { - SplitStringByDelimeter(str, delimeter, result); - return Status::OK(); - } - - std::string process_str = str; - while (index != std::string::npos) { - std::string prefix = process_str.substr(last, index - last); - std::string append_prefix; - if (!prefix.empty()) { - std::vector prefix_split; - SplitStringByDelimeter(prefix, delimeter, prefix_split); - for (size_t i = 0; i < prefix_split.size() - 1; i++) { - result.push_back(prefix_split[i]); - } - append_prefix = prefix_split[prefix_split.size() - 1]; - } - last = index + 1; - std::string postfix = process_str.substr(last); - index = postfix.find_first_of(quote, 0); - fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index", index = std::string::npos); - - if (index == std::string::npos) { - return Status(SERVER_UNEXPECTED_ERROR, ""); - } - std::string quoted_text = postfix.substr(0, index); - append_prefix += quoted_text; - - last = index + 1; - index = postfix.find_first_of(delimeter, last); - fiu_do_on("StringHelpFunctions.SplitStringByQuote.index_gt_last", last = 0); - fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index2", index = std::string::npos); - - if (index != std::string::npos) { - if (index > last) { - append_prefix += postfix.substr(last, index - last); - } - } else { - append_prefix += postfix.substr(last); - } - result.emplace_back(append_prefix); - fiu_do_on("StringHelpFunctions.SplitStringByQuote.last_is_end", last = postfix.length()); - - if (last == postfix.length()) { - return Status::OK(); - } - - process_str = postfix.substr(index + 1); - last = 0; - index = process_str.find_first_of(quote, last); - } - - if (!process_str.empty()) { - SplitStringByDelimeter(process_str, delimeter, result); - } - - return Status::OK(); -} - -bool -StringHelpFunctions::IsRegexMatch(const std::string& target_str, const std::string& pattern_str) { - // if target_str equals pattern_str, return true - if (target_str == pattern_str) { - return true; - } - - // regex match - // for illegal regex expression, the std::regex will throw exception, regard as unmatch - try { - std::regex pattern(pattern_str); - std::smatch results; - return std::regex_match(target_str, results, pattern); - } catch (std::exception& e) { - LOG_SERVER_ERROR_ << "Regex exception: " << e.what(); - } - - return false; -} - -Status -StringHelpFunctions::ConvertToBoolean(const std::string& str, bool& value) { - std::string s = str; - std::transform(s.begin(), s.end(), s.begin(), ::tolower); - value = s == "true" || s == "on" || s == "yes" || s == "1"; - - return Status::OK(); -} - -} // namespace milvus diff --git a/internal/core/src/utils/StringHelpFunctions.h b/internal/core/src/utils/StringHelpFunctions.h deleted file mode 100644 index bcb5c38052..0000000000 --- a/internal/core/src/utils/StringHelpFunctions.h +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include "utils/Status.h" - -#include -#include - -namespace milvus { - -class StringHelpFunctions { - private: - StringHelpFunctions() = default; - - public: - // trim blanks from begin and end - // " a b c " => "a b c" - static void - TrimStringBlank(std::string& string); - - // trim quotes from begin and end - // "'abc'" => "abc" - static void - TrimStringQuote(std::string& string, const std::string& qoute); - - // split string by delimeter ',' - // a,b,c a | b | c - // a,b, a | b | - // ,b,c | b | c - // ,b, | b | - // ,, | | - // a a - static void - SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector& result); - - // merge strings with delimeter - // "a", "b", "c" => "a,b,c" - static void - MergeStringWithDelimeter(const std::vector& strs, const std::string& delimeter, std::string& result); - - // assume the collection has two columns, quote='\"', delimeter=',' - // a,b a | b - // "aa,gg,yy",b aa,gg,yy | b - // aa"dd,rr"kk,pp aadd,rrkk | pp - // "aa,bb" aa,bb - // 55,1122\"aa,bb\",yyy,\"kkk\" 55 | 1122aa,bb | yyy | kkk - // "55,1122"aa,bb",yyy,"kkk" illegal - static Status - SplitStringByQuote(const std::string& str, - const std::string& delimeter, - const std::string& quote, - std::vector& result); - - // std regex match function - // regex grammar reference: http://www.cplusplus.com/reference/regex/ECMAScript/ - static bool - IsRegexMatch(const std::string& target_str, const std::string& pattern); - - // conversion rules refer to ValidationUtil::ValidateStringIsBool() - // "true", "on", "yes", "1" ==> true - // "false", "off", "no", "0", "" ==> false - static Status - ConvertToBoolean(const std::string& str, bool& value); -}; - -} // namespace milvus diff --git a/internal/core/src/utils/ThreadPool.h b/internal/core/src/utils/ThreadPool.h deleted file mode 100644 index 1e6f0f966c..0000000000 --- a/internal/core/src/utils/ThreadPool.h +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define MAX_THREADS_NUM 32 - -namespace milvus { - -class ThreadPool { - public: - explicit ThreadPool(size_t threads, size_t queue_size = 1000); - - template - auto - enqueue(F&& f, Args&&... args) -> std::future::type>; - - void - Stop(); - - ~ThreadPool(); - - private: - // need to keep track of threads so we can join them - std::vector workers_; - - // the task queue - std::queue> tasks_; - - size_t max_queue_size_; - - // synchronization - std::mutex queue_mutex_; - - std::condition_variable condition_; - - std::atomic_bool stop_; -}; - -// the constructor just launches some amount of workers -inline ThreadPool::ThreadPool(size_t threads, size_t queue_size) : max_queue_size_(queue_size), stop_(false) { - for (size_t i = 0; i < threads; ++i) - workers_.emplace_back([this] { - for (;;) { - std::function task; - - { - std::unique_lock lock(this->queue_mutex_); - this->condition_.wait(lock, [this] { return this->stop_ || !this->tasks_.empty(); }); - if (this->stop_ && this->tasks_.empty()) - return; - task = std::move(this->tasks_.front()); - this->tasks_.pop(); - } - this->condition_.notify_all(); - - task(); - } - }); -} - -// add new work item to the pool -template -auto -ThreadPool::enqueue(F&& f, Args&&... args) -> std::future::type> { - using return_type = typename std::result_of::type; - - auto task = - std::make_shared>(std::bind(std::forward(f), std::forward(args)...)); - fiu_do_on("ThreadPool.enqueue.stop_is_true", stop_ = true); - std::future res = task->get_future(); - { - std::unique_lock lock(queue_mutex_); - this->condition_.wait(lock, [this] { return this->tasks_.size() < max_queue_size_; }); - // don't allow enqueueing after stopping the pool - if (stop_) - throw std::runtime_error("enqueue on stopped ThreadPool"); - - tasks_.emplace([task]() { (*task)(); }); - } - condition_.notify_all(); - return res; -} - -inline void -ThreadPool::Stop() { - if (stop_) { - return; - } - { - std::unique_lock lock(queue_mutex_); - stop_ = true; - } - condition_.notify_all(); - for (std::thread& worker : workers_) { - worker.join(); - } -} - -// the destructor joins all threads -inline ThreadPool::~ThreadPool() { - Stop(); -} - -using ThreadPoolPtr = std::shared_ptr; - -} // namespace milvus diff --git a/internal/core/src/utils/TimeRecorder.cpp b/internal/core/src/utils/TimeRecorder.cpp deleted file mode 100644 index f4d6bff80b..0000000000 --- a/internal/core/src/utils/TimeRecorder.cpp +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include - -#include "utils/Log.h" -#include "utils/TimeRecorder.h" - -namespace milvus { - -TimeRecorder::TimeRecorder(std::string hdr, int64_t log_level) : header_(std::move(hdr)), log_level_(log_level) { - start_ = last_ = stdclock::now(); -} - -std::string -TimeRecorder::GetTimeSpanStr(double span) { - std::string str_sec = std::to_string(span * 0.000001) + ((span > 1000000) ? " seconds" : " second"); - std::string str_ms = std::to_string(span * 0.001) + " ms"; - - return str_sec + " [" + str_ms + "]"; -} - -void -TimeRecorder::PrintTimeRecord(const std::string& msg, double span) { - std::string str_log; - if (!header_.empty()) { - str_log += header_ + ": "; - } - str_log += msg; - str_log += " ("; - str_log += TimeRecorder::GetTimeSpanStr(span); - str_log += ")"; - - switch (log_level_) { - case 0: - LOG_SERVER_TRACE_ << str_log; - break; - case 1: - LOG_SERVER_DEBUG_ << str_log; - break; - case 2: - LOG_SERVER_INFO_ << str_log; - break; - case 3: - LOG_SERVER_WARNING_ << str_log; - break; - case 4: - LOG_SERVER_ERROR_ << str_log; - break; - case 5: - LOG_SERVER_FATAL_ << str_log; - break; - default: - LOG_SERVER_INFO_ << str_log; - break; - } -} - -double -TimeRecorder::RecordSection(const std::string& msg) { - stdclock::time_point curr = stdclock::now(); - double span = (std::chrono::duration(curr - last_)).count(); - last_ = curr; - - PrintTimeRecord(msg, span); - return span; -} - -double -TimeRecorder::ElapseFromBegin(const std::string& msg) { - stdclock::time_point curr = stdclock::now(); - double span = (std::chrono::duration(curr - start_)).count(); - - PrintTimeRecord(msg, span); - return span; -} - -TimeRecorderAuto::TimeRecorderAuto(std::string hdr, int64_t log_level) : TimeRecorder(hdr, log_level) { -} - -TimeRecorderAuto::~TimeRecorderAuto() { - ElapseFromBegin("totally cost"); -} - -} // namespace milvus diff --git a/internal/core/src/utils/TimeRecorder.h b/internal/core/src/utils/TimeRecorder.h deleted file mode 100644 index ffab6c5b88..0000000000 --- a/internal/core/src/utils/TimeRecorder.h +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include "utils/Log.h" - -namespace milvus { - -inline void -print_timestamp(const std::string& message) { - std::chrono::time_point now = std::chrono::system_clock::now(); - auto duration = now.time_since_epoch(); - auto micros = std::chrono::duration_cast(duration).count(); - micros %= 1000000; - double millisecond = (double)micros / 1000.0; - - LOG_SERVER_DEBUG_ << std::fixed << " " << millisecond << "(ms) [timestamp]" << message; -} - -class TimeRecorder { - using stdclock = std::chrono::high_resolution_clock; - - public: - // trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5 - explicit TimeRecorder(std::string hdr, int64_t log_level = 1); - virtual ~TimeRecorder() = default; - - double - RecordSection(const std::string& msg); - - double - ElapseFromBegin(const std::string& msg); - - static std::string - GetTimeSpanStr(double span); - - private: - void - PrintTimeRecord(const std::string& msg, double span); - - private: - std::string header_; - stdclock::time_point start_; - stdclock::time_point last_; - int64_t log_level_; -}; - -class TimeRecorderAuto : public TimeRecorder { - public: - explicit TimeRecorderAuto(std::string hdr, int64_t log_level = 1); - ~TimeRecorderAuto() override; -}; - -} // namespace milvus diff --git a/internal/core/src/utils/TimerContext.h b/internal/core/src/utils/TimerContext.h deleted file mode 100644 index 0396c4ef61..0000000000 --- a/internal/core/src/utils/TimerContext.h +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "utils/Log.h" -#include "utils/ThreadPool.h" - -namespace milvus { - -struct TimerContext { - using HandlerT = std::function; - struct Context { - /* Context(int interval_us, HandlerT& handler, ThreadPoolPtr pool = nullptr) */ - /* : interval_(interval_us), handler_(handler), timer_(io, interval_), pool_(pool) { */ - /* } */ - int interval_us; - HandlerT handler; - ThreadPoolPtr pool = nullptr; - }; - - TimerContext(boost::asio::io_service& io, int interval_us, HandlerT& handler, ThreadPoolPtr pool) - : io_(io), interval_(interval_us), handler_(handler), timer_(io, interval_), pool_(pool) { - } - TimerContext(boost::asio::io_service& io, Context& context) - : io_(io), - interval_(context.interval_us), - handler_(context.handler), - timer_(io, interval_), - pool_(context.pool) { - } - - void - Reschedule(const boost::system::error_code& ec); - - boost::asio::io_service& io_; - boost::posix_time::microseconds interval_; - boost::asio::deadline_timer timer_; - HandlerT handler_; - ThreadPoolPtr pool_; -}; - -inline void -TimerContext::Reschedule(const boost::system::error_code& ec) { - try { - pool_->enqueue(handler_, ec); - } catch (std::exception& ex) { - LOG_SERVER_ERROR_ << "Fail to enqueue handler: " << std::string(ex.what()); - } - boost::system::error_code e; - auto new_expires = timer_.expires_at() + interval_; - timer_.expires_at(new_expires, e); - if (e) { - LOG_SERVER_ERROR_ << "Fail to Reschedule: " << e; - } - timer_.async_wait(std::bind(&TimerContext::Reschedule, this, std::placeholders::_1)); -} - -using TimerContextPtr = std::shared_ptr; - -} // namespace milvus diff --git a/internal/core/src/utils/TimerManager.cpp b/internal/core/src/utils/TimerManager.cpp deleted file mode 100644 index a52893c86f..0000000000 --- a/internal/core/src/utils/TimerManager.cpp +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include - -#include "utils/Log.h" -#include "utils/TimerManager.h" - -namespace milvus { - -TimerManager::TimerManager(unsigned int pool_size) : pool_size_(pool_size) { -} - -TimerManager::~TimerManager() { -} - -Status -TimerManager::SetPoolSize(unsigned int pool_size) { - if (timer_exeutors_) { - return Status(SERVER_UNEXPECTED_ERROR, "Cannot set pool size since it has been set already"); - } - pool_size_ = pool_size; - return Status::OK(); -} - -Status -TimerManager::Run() { - boost::system::error_code ec; - io_.run(ec); - Status status; - if (ec) { - status = Status(SERVER_UNEXPECTED_ERROR, ec.message()); - } - return status; -} - -Status -TimerManager::Start() { - for (auto& timer : timers_) { - timer->timer_.async_wait(std::bind(&TimerContext::Reschedule, timer, std::placeholders::_1)); - } - return Status::OK(); -} - -void -TimerManager::Stop() { - boost::system::error_code ec; - for (auto& timer : timers_) { - timer->timer_.cancel(ec); - if (ec) { - LOG_SERVER_ERROR_ << "Fail to cancel timer: " << ec; - } - } - if (timer_exeutors_) { - timer_exeutors_->Stop(); - } -} - -void -TimerManager::AddTimer(int interval_us, TimerContext::HandlerT handler) { - if (!timer_exeutors_) { - timer_exeutors_ = std::make_shared(pool_size_); - } - timers_.emplace_back(std::make_shared(io_, interval_us, handler, timer_exeutors_)); -} - -void -TimerManager::AddTimer(const TimerContext::Context& ctx) { - if (!timer_exeutors_) { - timer_exeutors_ = std::make_shared(pool_size_); - } - TimerContext::Context context(ctx); - context.pool = timer_exeutors_; - timers_.emplace_back(std::make_shared(io_, context)); -} - -} // namespace milvus diff --git a/internal/core/src/utils/TimerManager.h b/internal/core/src/utils/TimerManager.h deleted file mode 100644 index 5e5441faae..0000000000 --- a/internal/core/src/utils/TimerManager.h +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include -#include - -#include "utils/Status.h" -#include "utils/ThreadPool.h" -#include "utils/TimerContext.h" - -namespace milvus { - -class TimerManager { - public: - TimerManager() = default; - explicit TimerManager(unsigned int pool_size); - - Status - SetPoolSize(unsigned int pool_size); - - void - AddTimer(int interval_us, TimerContext::HandlerT handler); - void - AddTimer(const TimerContext::Context& ctx); - - virtual Status - Run(); - - virtual Status - Start(); - virtual void - Stop(); - - virtual ~TimerManager(); - - protected: - boost::asio::io_service io_; - ThreadPoolPtr timer_exeutors_; - unsigned int pool_size_; - std::vector timers_; -}; - -using TimerManagerPtr = std::shared_ptr; - -} // namespace milvus