diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 5ff5b5041b..b7cf014f63 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -10,6 +10,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-577 - Unittest Query randomly hung - MS-587 - Count get wrong result after adding vectors and index built immediately - MS-599 - search wrong result when table created with metric_type: IP +- MS-601 - Docker logs error caused by get CPUTemperature error ## Improvement - MS-552 - Add and change the easylogging library @@ -28,6 +29,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-609 - Update task construct function - MS-611 - Add resources validity check in ResourceMgr - MS-619 - Add optimizer class in scheduler +- MS-614 - Preload table at startup ## New Feature diff --git a/cpp/README.md b/cpp/README.md index f5f77d25f8..9211a5d876 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -105,13 +105,20 @@ please reinstall CMake with curl: ``` ##### code format and linting - +Install clang-format and clang-tidy ```shell CentOS 7: $ yum install clang -Ubuntu 16.04 or 18.04: -$ sudo apt-get install clang-format clang-tidy - +Ubuntu 16.04: +$ sudo apt-get install clang-tidy +$ wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - +$ sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main" +$ sudo apt-get update +$ sudo apt-get install clang-format-6.0 +Ubuntu 18.04: +$ sudo apt-get install clang-tidy clang-format +``` +```shell $ ./build.sh -l ``` @@ -122,13 +129,14 @@ $ ./build.sh -u ``` ##### Run code coverage - +Install lcov ```shell CentOS 7: $ yum install lcov Ubuntu 16.04 or 18.04: $ sudo apt-get install lcov - +``` +```shell $ ./build.sh -u -c ``` diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 3d63cdfafa..2f2f699e09 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -18,6 +18,9 @@ db_config: # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory build_index_gpu: 0 # gpu id used for building index + preload_table: # preload data at startup, '*' means load all tables, empty value means no preload + # you can specify preload tables like this: table1,table2,table3 + metric_config: enable_monitor: false # enable monitoring or not collector: prometheus # prometheus diff --git a/cpp/coverage.sh b/cpp/coverage.sh index c0defe6f5a..8a7e5f52a1 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -104,7 +104,7 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ "src/metrics/MetricBase.h"\ "src/server/Server.cpp"\ "src/server/DBWrapper.cpp"\ - "src/server/grpc_impl/GrpcMilvusServer.cpp"\ + "src/server/grpc_impl/GrpcServer.cpp"\ "src/utils/easylogging++.h"\ "src/utils/easylogging++.cc"\ diff --git a/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp index f4b1a7953f..b4c3910a01 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp @@ -76,7 +76,7 @@ ConvertToDataset(std::vector query_results) { auto p_id = (int64_t*)malloc(sizeof(int64_t) * elems); auto p_dist = (float*)malloc(sizeof(float) * elems); -// TODO: throw if malloc failed. + // TODO: throw if malloc failed. #pragma omp parallel for for (auto i = 0; i < query_results.size(); ++i) { diff --git a/cpp/src/core/knowhere/knowhere/common/Buffer.h b/cpp/src/core/knowhere/knowhere/common/Buffer.h index d95da18602..f9e15d95bd 100644 --- a/cpp/src/core/knowhere/knowhere/common/Buffer.h +++ b/cpp/src/core/knowhere/knowhere/common/Buffer.h @@ -36,7 +36,7 @@ struct BufferDeleter { free((void*)buffer->data()); } }; -} +} // namespace internal inline BufferPtr MakeBufferSmart(uint8_t* data, const int64_t size) { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp index 4fc9a82009..2612a63630 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include #include #include #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/FaissBaseIndex.h" +#include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/helpers/FaissIO.h" namespace knowhere { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp index dc886b7383..d538c0dea1 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -#include #include #include #include @@ -26,6 +25,7 @@ #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexIVFPQ.h" #include "knowhere/index/vector_index/helpers/Cloner.h" #include "knowhere/index/vector_index/helpers/FaissIO.h" diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp index 077f0817da..213141b3ac 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp @@ -23,6 +23,7 @@ #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/IndexGPUIVFPQ.h" +#include "knowhere/index/vector_index/IndexIVFPQ.h" namespace knowhere { diff --git a/cpp/src/core/unittest/test_ivf.cpp b/cpp/src/core/unittest/test_ivf.cpp index a0da0eca8e..17eb888ddc 100644 --- a/cpp/src/core/unittest/test_ivf.cpp +++ b/cpp/src/core/unittest/test_ivf.cpp @@ -43,9 +43,9 @@ namespace kn = knowhere; } // namespace +using ::testing::Combine; using ::testing::TestWithParam; using ::testing::Values; -using ::testing::Combine; constexpr int device_id = 0; constexpr int64_t DIM = 128; diff --git a/cpp/src/core/unittest/test_kdt.cpp b/cpp/src/core/unittest/test_kdt.cpp index 6fa6ba33f3..f9e02bd9a4 100644 --- a/cpp/src/core/unittest/test_kdt.cpp +++ b/cpp/src/core/unittest/test_kdt.cpp @@ -34,9 +34,9 @@ namespace kn = knowhere; } // namespace +using ::testing::Combine; using ::testing::TestWithParam; using ::testing::Values; -using ::testing::Combine; class KDTTest : public DataGen, public ::testing::Test { protected: diff --git a/cpp/src/core/unittest/test_nsg/test_nsg.cpp b/cpp/src/core/unittest/test_nsg/test_nsg.cpp index 70261a63dc..b59f0d4928 100644 --- a/cpp/src/core/unittest/test_nsg/test_nsg.cpp +++ b/cpp/src/core/unittest/test_nsg/test_nsg.cpp @@ -32,9 +32,9 @@ namespace kn = knowhere; } // namespace +using ::testing::Combine; using ::testing::TestWithParam; using ::testing::Values; -using ::testing::Combine; constexpr int64_t DEVICE_ID = 1; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index f5560a5476..ce3463499b 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -207,7 +207,7 @@ DBImpl::PreloadTable(const std::string &table_id) { size += engine->PhysicalSize(); if (size > available_size) { - break; + return Status(SERVER_CACHE_FULL, "Cache is full"); } else { try { // step 1: load index @@ -297,7 +297,8 @@ DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) { std::vector file_types; if (index.engine_type_ == static_cast(EngineType::FAISS_IDMAP)) { file_types = { - static_cast(meta::TableFileSchema::NEW), static_cast(meta::TableFileSchema::NEW_MERGE), + static_cast(meta::TableFileSchema::NEW), + static_cast(meta::TableFileSchema::NEW_MERGE), }; } else { file_types = { @@ -640,8 +641,9 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date, const m ENGINE_LOG_DEBUG << "Merging file " << file_schema.file_id_; index_size = index->Size(); - if (index_size >= file_schema.index_file_size_) + if (index_size >= file_schema.index_file_size_) { break; + } } // step 3: serialize to disk diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 5827513297..2c877c78ed 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -55,9 +55,9 @@ MemTableFile::CreateTableFile() { Status MemTableFile::Add(const VectorSourcePtr& source, IDNumbers& vector_ids) { if (table_file_schema_.dimension_ <= 0) { - std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + - std::to_string(table_file_schema_.dimension_) + ", table_id = " + - table_file_schema_.table_id_; + std::string err_msg = + "MemTableFile::Add: table_file_schema dimension = " + std::to_string(table_file_schema_.dimension_) + + ", table_id = " + table_file_schema_.table_id_; ENGINE_LOG_ERROR << err_msg; return Status(DB_ERROR, "Not able to create table file"); } diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 872c1ced2e..f9f1569a65 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -148,15 +148,18 @@ static const MetaSchema TABLES_SCHEMA(META_TABLES, { }); // TableFiles schema -static const MetaSchema TABLEFILES_SCHEMA( - META_TABLEFILES, - { - MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), MetaField("table_id", "VARCHAR(255)", "NOT NULL"), - MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), MetaField("file_id", "VARCHAR(255)", "NOT NULL"), - MetaField("file_type", "INT", "DEFAULT 0 NOT NULL"), MetaField("file_size", "BIGINT", "DEFAULT 0 NOT NULL"), - MetaField("row_count", "BIGINT", "DEFAULT 0 NOT NULL"), MetaField("updated_time", "BIGINT", "NOT NULL"), - MetaField("created_on", "BIGINT", "NOT NULL"), MetaField("date", "INT", "DEFAULT -1 NOT NULL"), - }); +static const MetaSchema TABLEFILES_SCHEMA(META_TABLEFILES, { + MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), + MetaField("table_id", "VARCHAR(255)", "NOT NULL"), + MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), + MetaField("file_id", "VARCHAR(255)", "NOT NULL"), + MetaField("file_type", "INT", "DEFAULT 0 NOT NULL"), + MetaField("file_size", "BIGINT", "DEFAULT 0 NOT NULL"), + MetaField("row_count", "BIGINT", "DEFAULT 0 NOT NULL"), + MetaField("updated_time", "BIGINT", "NOT NULL"), + MetaField("created_on", "BIGINT", "NOT NULL"), + MetaField("date", "INT", "DEFAULT -1 NOT NULL"), + }); } // namespace diff --git a/cpp/src/metrics/SystemInfo.cpp b/cpp/src/metrics/SystemInfo.cpp index 1414d94eae..a49c3070a6 100644 --- a/cpp/src/metrics/SystemInfo.cpp +++ b/cpp/src/metrics/SystemInfo.cpp @@ -16,6 +16,7 @@ // under the License. #include "metrics/SystemInfo.h" +#include "utils/Log.h" #include #include @@ -24,6 +25,9 @@ #include #include #include +#include +#include +#include namespace milvus { namespace server { @@ -60,12 +64,12 @@ SystemInfo::Init() { nvmlReturn_t nvmlresult; nvmlresult = nvmlInit(); if (NVML_SUCCESS != nvmlresult) { - printf("System information initilization failed"); + SERVER_LOG_ERROR << "System information initilization failed"; return; } nvmlresult = nvmlDeviceGetCount(&num_device_); if (NVML_SUCCESS != nvmlresult) { - printf("Unable to get devidce number"); + SERVER_LOG_ERROR << "Unable to get devidce number"; return; } @@ -151,7 +155,7 @@ SystemInfo::getTotalCpuTime(std::vector& work_time_array) { std::vector total_time_array; FILE* file = fopen("/proc/stat", "r"); if (file == NULL) { - perror("Could not open stat file"); + SERVER_LOG_ERROR << "Could not open stat file"; return total_time_array; } @@ -162,7 +166,7 @@ SystemInfo::getTotalCpuTime(std::vector& work_time_array) { char buffer[1024]; char* ret = fgets(buffer, sizeof(buffer) - 1, file); if (ret == NULL) { - perror("Could not read stat file"); + SERVER_LOG_ERROR << "Could not read stat file"; fclose(file); return total_time_array; } @@ -237,18 +241,39 @@ SystemInfo::GPUTemperature() { std::vector SystemInfo::CPUTemperature() { std::vector result; - for (int i = 0; i <= num_physical_processors_; ++i) { - std::string path = "/sys/class/thermal/thermal_zone" + std::to_string(i) + "/temp"; - FILE* file = fopen(path.data(), "r"); - if (file == nullptr) { - perror("Could not open thermal file"); - return result; - } - float temp; - fscanf(file, "%f", &temp); - result.push_back(temp / 1000); - fclose(file); + std::string path = "/sys/class/hwmon/"; + + DIR *dir = NULL; + dir = opendir(path.c_str()); + if (!dir) { + SERVER_LOG_ERROR << "Could not open hwmon directory"; + return result; } + + struct dirent *ptr = NULL; + while ((ptr = readdir(dir)) != NULL) { + std::string filename(path); + filename.append(ptr->d_name); + + char buf[100]; + if (readlink(filename.c_str(), buf, 100) != -1) { + std::string m(buf); + if (m.find("coretemp") != std::string::npos) { + std::string object = filename; + object += "/temp1_input"; + FILE *file = fopen(object.c_str(), "r"); + if (file == nullptr) { + SERVER_LOG_ERROR << "Could not open temperature file" + exit(1); + } + float temp; + fscanf(file, "%f", &temp); + result.push_back(temp / 1000); + } + } + } + closedir(dir); + return result; } std::vector diff --git a/cpp/src/scheduler/optimizer/HybridPass.cpp b/cpp/src/scheduler/optimizer/HybridPass.cpp index 1b0ab7c4d0..f172a7beb9 100644 --- a/cpp/src/scheduler/optimizer/HybridPass.cpp +++ b/cpp/src/scheduler/optimizer/HybridPass.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "HybridPass.h" +#include "scheduler/optimizer/HybridPass.h" #include "scheduler/task/SearchTask.h" namespace milvus { @@ -24,12 +24,12 @@ namespace scheduler { bool HybridPass::Run(const TaskPtr& task) { // TODO: Index::IVFSQ8Hybrid, if nq < threshold set cpu, else set gpu - if (task->Type() != TaskType::SearchTask) return false; + if (task->Type() != TaskType::SearchTask) + return false; auto search_task = std::static_pointer_cast(task); // if (search_task->file_->engine_type_ == engine::EngineType::FAISS_IVFSQ8Hybrid) return false; } -} -} - +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/HybridPass.h b/cpp/src/scheduler/optimizer/HybridPass.h index e043bf8ad5..0d02a8bda9 100644 --- a/cpp/src/scheduler/optimizer/HybridPass.h +++ b/cpp/src/scheduler/optimizer/HybridPass.h @@ -16,16 +16,16 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include #include "Pass.h" @@ -43,6 +43,5 @@ class HybridPass : public Pass { using HybridPassPtr = std::shared_ptr; -} -} - +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/Optimizer.cpp b/cpp/src/scheduler/optimizer/Optimizer.cpp index 517cd85f79..c5fa311a27 100644 --- a/cpp/src/scheduler/optimizer/Optimizer.cpp +++ b/cpp/src/scheduler/optimizer/Optimizer.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "Optimizer.h" +#include "scheduler/optimizer/Optimizer.h" namespace milvus { namespace scheduler { @@ -38,6 +38,5 @@ Optimizer::Run(const TaskPtr& task) { return false; } -} -} - +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/Optimizer.h b/cpp/src/scheduler/optimizer/Optimizer.h index cbb13a8fc2..99282e66a6 100644 --- a/cpp/src/scheduler/optimizer/Optimizer.h +++ b/cpp/src/scheduler/optimizer/Optimizer.h @@ -16,16 +16,16 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include #include "Pass.h" @@ -40,12 +40,11 @@ class Optimizer { Init(); bool - Run(const TaskPtr &task); + Run(const TaskPtr& task); private: std::vector pass_list_; }; -} -} - +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/Pass.h b/cpp/src/scheduler/optimizer/Pass.h index 1c8def41a2..959c3ea5ee 100644 --- a/cpp/src/scheduler/optimizer/Pass.h +++ b/cpp/src/scheduler/optimizer/Pass.h @@ -16,16 +16,16 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include #include "scheduler/task/Task.h" @@ -35,12 +35,13 @@ namespace scheduler { class Pass { public: virtual void - Init() {} + Init() { + } virtual bool Run(const TaskPtr& task) = 0; }; using PassPtr = std::shared_ptr; -} -} +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/task/SearchTask.cpp b/cpp/src/scheduler/task/SearchTask.cpp index ee24dace43..9925a8bcf8 100644 --- a/cpp/src/scheduler/task/SearchTask.cpp +++ b/cpp/src/scheduler/task/SearchTask.cpp @@ -157,8 +157,8 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { size_t file_size = index_engine_->PhysicalSize(); - std::string info = "Load file id:" + std::to_string(file_->id_) + " file type:" + - std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + + std::string info = "Load file id:" + std::to_string(file_->id_) + + " file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + " bytes from location: " + file_->location_ + " totally cost"; double span = rc.ElapseFromBegin(info); // for (auto &context : search_contexts_) { diff --git a/cpp/src/sdk/include/Status.h b/cpp/src/sdk/include/Status.h index a81116b31d..008f9956d2 100644 --- a/cpp/src/sdk/include/Status.h +++ b/cpp/src/sdk/include/Status.h @@ -20,12 +20,12 @@ #include /** \brief Milvus SDK namespace -*/ + */ namespace milvus { /** -* @brief Status Code for SDK interface return -*/ + * @brief Status Code for SDK interface return + */ enum class StatusCode { OK = 0, @@ -41,8 +41,8 @@ enum class StatusCode { }; /** -* @brief Status for SDK interface return -*/ + * @brief Status for SDK interface return + */ class Status { public: Status(StatusCode code, const std::string& msg); diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 5d6a237c1d..d383eb5edd 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -655,294 +655,62 @@ Config::SetConfigValueInMem(const std::string& parent_key, const std::string& ch } //////////////////////////////////////////////////////////////////////////////// -/* server config */ std::string -Config::GetServerConfigStrAddress() { +Config::GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value) { std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); + if (!GetConfigValueInMem(parent_key, child_key, value).ok()) { + value = GetConfigNode(parent_key).GetValue(child_key, default_value); + SetConfigValueInMem(parent_key, child_key, value); } return value; } -std::string -Config::GetServerConfigStrPort() { - std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); - } - return value; -} - -std::string -Config::GetServerConfigStrDeployMode() { - std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_DEPLOY_MODE, CONFIG_SERVER_DEPLOY_MODE_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); - } - return value; -} - -std::string -Config::GetServerConfigStrTimeZone() { - std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); - } - return value; -} - -//////////////////////////////////////////////////////////////////////////////// -/* db config */ -std::string -Config::GetDBConfigStrPrimaryPath() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PRIMARY_PATH, CONFIG_DB_PRIMARY_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); - } - return value; -} - -std::string -Config::GetDBConfigStrSecondaryPath() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SECONDARY_PATH, CONFIG_DB_SECONDARY_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); - } - return value; -} - -std::string -Config::GetDBConfigStrBackendUrl() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL, CONFIG_DB_BACKEND_URL_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); - } - return value; -} - -std::string -Config::GetDBConfigStrArchiveDiskThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, - CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); - } - return value; -} - -std::string -Config::GetDBConfigStrArchiveDaysThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, - CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); - } - return value; -} - -std::string -Config::GetDBConfigStrInsertBufferSize() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); - } - return value; -} - -std::string -Config::GetDBConfigStrBuildIndexGPU() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); - } - return value; -} - -//////////////////////////////////////////////////////////////////////////////// -/* metric config */ -std::string -Config::GetMetricConfigStrEnableMonitor() { - std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value).ok()) { - value = - GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_ENABLE_MONITOR, CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); - } - return value; -} - -std::string -Config::GetMetricConfigStrCollector() { - std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { - value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); - } - return value; -} - -std::string -Config::GetMetricConfigStrPrometheusPort() { - std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { - value = - GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); - } - return value; -} - -//////////////////////////////////////////////////////////////////////////////// -/* cache config */ -std::string -Config::GetCacheConfigStrCpuCacheCapacity() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value).ok()) { - value = GetConfigNode(CONFIG_CACHE) - .GetValue(CONFIG_CACHE_CPU_CACHE_CAPACITY, CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); - } - return value; -} - -std::string -Config::GetCacheConfigStrCpuCacheThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_CACHE) - .GetValue(CONFIG_CACHE_CPU_CACHE_THRESHOLD, CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); - } - return value; -} - -std::string -Config::GetCacheConfigStrGpuCacheCapacity() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value).ok()) { - value = GetConfigNode(CONFIG_CACHE) - .GetValue(CONFIG_CACHE_GPU_CACHE_CAPACITY, CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); - } - return value; -} - -std::string -Config::GetCacheConfigStrGpuCacheThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_CACHE) - .GetValue(CONFIG_CACHE_GPU_CACHE_THRESHOLD, CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); - } - return value; -} - -std::string -Config::GetCacheConfigStrCacheInsertData() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { - value = GetConfigNode(CONFIG_CACHE) - .GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); - } - return value; -} - -//////////////////////////////////////////////////////////////////////////////// -/* engine config */ -std::string -Config::GetEngineConfigStrUseBlasThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_ENGINE) - .GetValue(CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); - } - return value; -} - -std::string -Config::GetEngineConfigStrOmpThreadNum() { - std::string value; - if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { - value = - GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); - } - return value; -} - -//////////////////////////////////////////////////////////////////////////////// -/* resource config */ -std::string -Config::GetResourceConfigStrMode() { - std::string value; - if (!GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { - value = GetConfigNode(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); - } - return value; -} - -//////////////////////////////////////////////////////////////////////////////// Status Config::GetServerConfigAddress(std::string& value) { - value = GetServerConfigStrAddress(); + value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); return CheckServerConfigAddress(value); } Status Config::GetServerConfigPort(std::string& value) { - value = GetServerConfigStrPort(); + value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); return CheckServerConfigPort(value); } Status Config::GetServerConfigDeployMode(std::string& value) { - value = GetServerConfigStrDeployMode(); + value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, CONFIG_SERVER_DEPLOY_MODE_DEFAULT); return CheckServerConfigDeployMode(value); } Status Config::GetServerConfigTimeZone(std::string& value) { - value = GetServerConfigStrTimeZone(); + value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); return CheckServerConfigTimeZone(value); } Status Config::GetDBConfigPrimaryPath(std::string& value) { - value = GetDBConfigStrPrimaryPath(); + value = GetConfigStr(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, CONFIG_DB_PRIMARY_PATH_DEFAULT); return CheckDBConfigPrimaryPath(value); } Status Config::GetDBConfigSecondaryPath(std::string& value) { - value = GetDBConfigStrSecondaryPath(); + value = GetConfigStr(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, CONFIG_DB_SECONDARY_PATH_DEFAULT); return Status::OK(); } Status Config::GetDBConfigBackendUrl(std::string& value) { - value = GetDBConfigStrBackendUrl(); + value = GetConfigStr(CONFIG_DB, CONFIG_DB_BACKEND_URL, CONFIG_DB_BACKEND_URL_DEFAULT); return CheckDBConfigBackendUrl(value); } Status Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { - std::string str = GetDBConfigStrArchiveDiskThreshold(); + std::string str = + GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) { return s; @@ -954,7 +722,8 @@ Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { Status Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { - std::string str = GetDBConfigStrArchiveDaysThreshold(); + std::string str = + GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) { return s; @@ -966,7 +735,7 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { Status Config::GetDBConfigInsertBufferSize(int32_t& value) { - std::string str = GetDBConfigStrInsertBufferSize(); + std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) { return s; @@ -978,7 +747,7 @@ Config::GetDBConfigInsertBufferSize(int32_t& value) { Status Config::GetDBConfigBuildIndexGPU(int32_t& value) { - std::string str = GetDBConfigStrBuildIndexGPU(); + std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); Status s = CheckDBConfigBuildIndexGPU(str); if (!s.ok()) { return s; @@ -988,9 +757,15 @@ Config::GetDBConfigBuildIndexGPU(int32_t& value) { return Status::OK(); } +Status +Config::GetDBConfigPreloadTable(std::string& value) { + value = GetConfigStr(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE); + return Status::OK(); +} + Status Config::GetMetricConfigEnableMonitor(bool& value) { - std::string str = GetMetricConfigStrEnableMonitor(); + std::string str = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); Status s = CheckMetricConfigEnableMonitor(str); if (!s.ok()) { return s; @@ -1003,19 +778,20 @@ Config::GetMetricConfigEnableMonitor(bool& value) { Status Config::GetMetricConfigCollector(std::string& value) { - value = GetMetricConfigStrCollector(); + value = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); return Status::OK(); } Status Config::GetMetricConfigPrometheusPort(std::string& value) { - value = GetMetricConfigStrPrometheusPort(); + value = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); return CheckMetricConfigPrometheusPort(value); } Status Config::GetCacheConfigCpuCacheCapacity(int32_t& value) { - std::string str = GetCacheConfigStrCpuCacheCapacity(); + std::string str = + GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); Status s = CheckCacheConfigCpuCacheCapacity(str); if (!s.ok()) { return s; @@ -1027,7 +803,8 @@ Config::GetCacheConfigCpuCacheCapacity(int32_t& value) { Status Config::GetCacheConfigCpuCacheThreshold(float& value) { - std::string str = GetCacheConfigStrCpuCacheThreshold(); + std::string str = + GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT); Status s = CheckCacheConfigCpuCacheThreshold(str); if (!s.ok()) { return s; @@ -1039,7 +816,8 @@ Config::GetCacheConfigCpuCacheThreshold(float& value) { Status Config::GetCacheConfigGpuCacheCapacity(int32_t& value) { - std::string str = GetCacheConfigStrGpuCacheCapacity(); + std::string str = + GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); Status s = CheckCacheConfigGpuCacheCapacity(str); if (!s.ok()) { return s; @@ -1051,7 +829,8 @@ Config::GetCacheConfigGpuCacheCapacity(int32_t& value) { Status Config::GetCacheConfigGpuCacheThreshold(float& value) { - std::string str = GetCacheConfigStrGpuCacheThreshold(); + std::string str = + GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); Status s = CheckCacheConfigGpuCacheThreshold(str); if (!s.ok()) { return s; @@ -1063,7 +842,8 @@ Config::GetCacheConfigGpuCacheThreshold(float& value) { Status Config::GetCacheConfigCacheInsertData(bool& value) { - std::string str = GetCacheConfigStrCacheInsertData(); + std::string str = + GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); Status s = CheckCacheConfigCacheInsertData(str); if (!s.ok()) { return s; @@ -1076,7 +856,8 @@ Config::GetCacheConfigCacheInsertData(bool& value) { Status Config::GetEngineConfigUseBlasThreshold(int32_t& value) { - std::string str = GetEngineConfigStrUseBlasThreshold(); + std::string str = + GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); Status s = CheckEngineConfigUseBlasThreshold(str); if (!s.ok()) { return s; @@ -1088,7 +869,7 @@ Config::GetEngineConfigUseBlasThreshold(int32_t& value) { Status Config::GetEngineConfigOmpThreadNum(int32_t& value) { - std::string str = GetEngineConfigStrOmpThreadNum(); + std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) { return s; @@ -1100,7 +881,7 @@ Config::GetEngineConfigOmpThreadNum(int32_t& value) { Status Config::GetResourceConfigMode(std::string& value) { - value = GetResourceConfigStrMode(); + value = GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); return CheckResourceConfigMode(value); } diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 9f6932d267..fb00498b96 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -56,6 +56,7 @@ static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4"; static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; +static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table"; /* cache config */ static const char* CONFIG_CACHE = "cache_config"; @@ -178,62 +179,8 @@ class Config { Status CheckResourceConfigPool(const std::vector& value); - /////////////////////////////////////////////////////////////////////////// - /* server config */ std::string - GetServerConfigStrAddress(); - std::string - GetServerConfigStrPort(); - std::string - GetServerConfigStrDeployMode(); - std::string - GetServerConfigStrTimeZone(); - - /* db config */ - std::string - GetDBConfigStrPrimaryPath(); - std::string - GetDBConfigStrSecondaryPath(); - std::string - GetDBConfigStrBackendUrl(); - std::string - GetDBConfigStrArchiveDiskThreshold(); - std::string - GetDBConfigStrArchiveDaysThreshold(); - std::string - GetDBConfigStrInsertBufferSize(); - std::string - GetDBConfigStrBuildIndexGPU(); - - /* metric config */ - std::string - GetMetricConfigStrEnableMonitor(); - std::string - GetMetricConfigStrCollector(); - std::string - GetMetricConfigStrPrometheusPort(); - - /* cache config */ - std::string - GetCacheConfigStrCpuCacheCapacity(); - std::string - GetCacheConfigStrCpuCacheThreshold(); - std::string - GetCacheConfigStrGpuCacheCapacity(); - std::string - GetCacheConfigStrGpuCacheThreshold(); - std::string - GetCacheConfigStrCacheInsertData(); - - /* engine config */ - std::string - GetEngineConfigStrUseBlasThreshold(); - std::string - GetEngineConfigStrOmpThreadNum(); - - /* resource config */ - std::string - GetResourceConfigStrMode(); + GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); public: /* server config */ @@ -261,6 +208,8 @@ class Config { GetDBConfigInsertBufferSize(int32_t& value); Status GetDBConfigBuildIndexGPU(int32_t& value); + Status + GetDBConfigPreloadTable(std::string& value); /* metric config */ Status diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 401c494575..306863f8ae 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -25,6 +25,7 @@ #include #include #include +#include namespace milvus { namespace server { @@ -155,6 +156,20 @@ DBWrapper::StartService() { db_->Start(); + // preload table + std::string preload_tables; + s = config.GetDBConfigPreloadTable(preload_tables); + if (!s.ok()) { + return s; + } + + s = PreloadTables(preload_tables); + if (!s.ok()) { + std::cerr << "ERROR! Failed to preload tables: " << preload_tables << std::endl; + std::cerr << s.ToString() << std::endl; + kill(0, SIGUSR1); + } + return Status::OK(); } @@ -167,5 +182,34 @@ DBWrapper::StopService() { return Status::OK(); } +Status +DBWrapper::PreloadTables(const std::string& preload_tables) { + if (preload_tables.empty()) { + // do nothing + } else if (preload_tables == "*") { + // load all tables + std::vector table_schema_array; + db_->AllTables(table_schema_array); + + for (auto& schema : table_schema_array) { + auto status = db_->PreloadTable(schema.table_id_); + if (!status.ok()) { + return status; + } + } + } else { + std::vector table_names; + StringHelpFunctions::SplitStringByDelimeter(preload_tables, ",", table_names); + for (auto& name : table_names) { + auto status = db_->PreloadTable(name); + if (!status.ok()) { + return status; + } + } + } + + return Status::OK(); +} + } // namespace server } // namespace milvus diff --git a/cpp/src/server/DBWrapper.h b/cpp/src/server/DBWrapper.h index 08e07c09f6..7016aa8805 100644 --- a/cpp/src/server/DBWrapper.h +++ b/cpp/src/server/DBWrapper.h @@ -21,6 +21,7 @@ #include "utils/Status.h" #include +#include namespace milvus { namespace server { @@ -52,6 +53,10 @@ class DBWrapper { return db_; } + private: + Status + PreloadTables(const std::string& preload_tables); + private: engine::DBPtr db_; }; diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.h b/cpp/src/server/grpc_impl/GrpcRequestHandler.h index 0decf61500..1a9b591154 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.h @@ -148,25 +148,25 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { ::milvus::grpc::TopKQueryResultList* response) override; /** - * @brief Internal use query interface - * - * This method is used to query vector in specified files. - * - * @param context, add context for every RPC - * @param request: - * file_id_array, specified files id array, queried. - * query_record_array, all vector are going to be queried. - * query_range_array, optional ranges for conditional search. If not specified, search whole table - * topk, how many similarity vectors will be searched. - * - * @param writer, write query result array. - * - * @return status - * - * @param context - * @param request - * @param writer - */ + * @brief Internal use query interface + * + * This method is used to query vector in specified files. + * + * @param context, add context for every RPC + * @param request: + * file_id_array, specified files id array, queried. + * query_record_array, all vector are going to be queried. + * query_range_array, optional ranges for conditional search. If not specified, search whole table + * topk, how many similarity vectors will be searched. + * + * @param writer, write query result array. + * + * @return status + * + * @param context + * @param request + * @param writer + */ ::grpc::Status SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResultList* response) override; diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp index b2aefe4f65..ac35f82947 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp @@ -36,7 +36,6 @@ ErrorMap(ErrorCode code) { {SERVER_INVALID_ARGUMENT, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, {SERVER_FILE_NOT_FOUND, ::milvus::grpc::ErrorCode::FILE_NOT_FOUND}, {SERVER_NOT_IMPLEMENT, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, - {SERVER_BLOCKING_QUEUE_EMPTY, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, {SERVER_CANNOT_CREATE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FOLDER}, {SERVER_CANNOT_CREATE_FILE, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FILE}, {SERVER_CANNOT_DELETE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_DELETE_FOLDER}, @@ -57,7 +56,7 @@ ErrorMap(ErrorCode code) { {SERVER_INVALID_INDEX_FILE_SIZE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, {SERVER_ILLEGAL_VECTOR_ID, ::milvus::grpc::ErrorCode::ILLEGAL_VECTOR_ID}, {SERVER_ILLEGAL_SEARCH_RESULT, ::milvus::grpc::ErrorCode::ILLEGAL_SEARCH_RESULT}, - {SERVER_CACHE_ERROR, ::milvus::grpc::ErrorCode::CACHE_FAILED}, + {SERVER_CACHE_FULL, ::milvus::grpc::ErrorCode::CACHE_FAILED}, {DB_META_TRANSACTION_FAILED, ::milvus::grpc::ErrorCode::META_FAILED}, {SERVER_BUILD_INDEX_ERROR, ::milvus::grpc::ErrorCode::BUILD_INDEX_ERROR}, {SERVER_OUT_OF_MEMORY, ::milvus::grpc::ErrorCode::OUT_OF_MEMORY}, diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index 9f0b8fd95d..dc2bd340eb 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -511,8 +511,8 @@ InsertTask::OnExecute() { } // step 6: update table flag - user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID : table_info.flag_ |= - engine::meta::FLAG_MASK_NO_USERID; + user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID + : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; status = DBWrapper::DB()->UpdateTableFlag(insert_param_->table_name(), table_info.flag_); #ifdef MILVUS_ENABLE_PROFILING diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index 9cba18ef41..dfc400ca9a 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -56,7 +56,6 @@ constexpr ErrorCode SERVER_NULL_POINTER = ToServerErrorCode(3); constexpr ErrorCode SERVER_INVALID_ARGUMENT = ToServerErrorCode(4); constexpr ErrorCode SERVER_FILE_NOT_FOUND = ToServerErrorCode(5); constexpr ErrorCode SERVER_NOT_IMPLEMENT = ToServerErrorCode(6); -constexpr ErrorCode SERVER_BLOCKING_QUEUE_EMPTY = ToServerErrorCode(7); constexpr ErrorCode SERVER_CANNOT_CREATE_FOLDER = ToServerErrorCode(8); constexpr ErrorCode SERVER_CANNOT_CREATE_FILE = ToServerErrorCode(9); constexpr ErrorCode SERVER_CANNOT_DELETE_FOLDER = ToServerErrorCode(10); @@ -74,7 +73,7 @@ constexpr ErrorCode SERVER_INVALID_ROWRECORD_ARRAY = ToServerErrorCode(107); constexpr ErrorCode SERVER_INVALID_TOPK = ToServerErrorCode(108); constexpr ErrorCode SERVER_ILLEGAL_VECTOR_ID = ToServerErrorCode(109); constexpr ErrorCode SERVER_ILLEGAL_SEARCH_RESULT = ToServerErrorCode(110); -constexpr ErrorCode SERVER_CACHE_ERROR = ToServerErrorCode(111); +constexpr ErrorCode SERVER_CACHE_FULL = ToServerErrorCode(111); constexpr ErrorCode SERVER_WRITE_ERROR = ToServerErrorCode(112); constexpr ErrorCode SERVER_INVALID_NPROBE = ToServerErrorCode(113); constexpr ErrorCode SERVER_INVALID_INDEX_NLIST = ToServerErrorCode(114); diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index ed2e1a446b..4a962f466c 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -31,7 +31,7 @@ static int warning_idx = 0; static int trace_idx = 0; static int error_idx = 0; static int fatal_idx = 0; -} +} // namespace // TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename void