From e102bddaa5b75cb83f72b51ad7df359455fef61e Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Wed, 9 Oct 2019 11:10:34 +0800 Subject: [PATCH 01/12] MS-601 Docker logs error caused by get CPUTemperature error Former-commit-id: 4611939ce3c9b66313aef12991b3477c43813ce7 --- cpp/src/metrics/PrometheusMetrics.cpp | 2 +- cpp/src/metrics/SystemInfo.cpp | 46 ++++++++++++++++++++------- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index bc1860389f..182f14d46c 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -46,7 +46,7 @@ PrometheusMetrics::Init() { return s.code(); } - const std::string uri = std::string("/tmp/metrics"); + const std::string uri = std::string("/metrics"); const std::size_t num_threads = 2; // Init Exposer diff --git a/cpp/src/metrics/SystemInfo.cpp b/cpp/src/metrics/SystemInfo.cpp index 1414d94eae..70e917ad04 100644 --- a/cpp/src/metrics/SystemInfo.cpp +++ b/cpp/src/metrics/SystemInfo.cpp @@ -24,6 +24,9 @@ #include #include #include +#include +#include +#include namespace milvus { namespace server { @@ -237,18 +240,39 @@ SystemInfo::GPUTemperature() { std::vector SystemInfo::CPUTemperature() { std::vector result; - for (int i = 0; i <= num_physical_processors_; ++i) { - std::string path = "/sys/class/thermal/thermal_zone" + std::to_string(i) + "/temp"; - FILE* file = fopen(path.data(), "r"); - if (file == nullptr) { - perror("Could not open thermal file"); - return result; - } - float temp; - fscanf(file, "%f", &temp); - result.push_back(temp / 1000); - fclose(file); + std::string path = "/sys/class/hwmon/"; + + DIR *dir = NULL; + dir = opendir(path.c_str()); + if (!dir) { + perror("opendir"); + return result; } + + struct dirent *ptr = NULL; + while ((ptr = readdir(dir)) != NULL) { + std::string filename(path); + filename.append(ptr->d_name); + + char buf[100]; + if (readlink(filename.c_str(), buf, 100) != -1) { + std::string m(buf); + if (m.find("coretemp") != std::string::npos) { + std::string object = filename; + object += "/temp1_input"; + FILE *file = fopen(object.c_str(), "r"); + if (file == nullptr) { + perror("Could not open temperature file"); + exit(1); + } + float temp; + fscanf(file, "%f", &temp); + result.push_back(temp / 1000); + } + } + } + closedir(dir); + return result; } std::vector From f97ba05661376d5bcff4f0444fc12a6f05b6c283 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 9 Oct 2019 11:49:27 +0800 Subject: [PATCH 02/12] MS-614 Preload table at startup Former-commit-id: 15ea4eb6d6c6f3437979a8b741a07bee5eaada5b --- cpp/CHANGELOG.md | 1 + cpp/conf/server_config.template | 3 ++ cpp/src/db/DBImpl.cpp | 5 ++- cpp/src/server/Config.cpp | 16 +++++++ cpp/src/server/Config.h | 5 +++ cpp/src/server/DBWrapper.cpp | 43 +++++++++++++++++++ cpp/src/server/DBWrapper.h | 4 ++ .../server/grpc_impl/GrpcRequestScheduler.cpp | 3 +- cpp/src/utils/Error.h | 3 +- 9 files changed, 77 insertions(+), 6 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 5ff5b5041b..8f3079b8dd 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -28,6 +28,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-609 - Update task construct function - MS-611 - Add resources validity check in ResourceMgr - MS-619 - Add optimizer class in scheduler +- MS-614 - Preload table at startup ## New Feature diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 3d63cdfafa..2f2f699e09 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -18,6 +18,9 @@ db_config: # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory build_index_gpu: 0 # gpu id used for building index + preload_table: # preload data at startup, '*' means load all tables, empty value means no preload + # you can specify preload tables like this: table1,table2,table3 + metric_config: enable_monitor: false # enable monitoring or not collector: prometheus # prometheus diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index ece62284f7..93f71dc737 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -206,7 +206,7 @@ DBImpl::PreloadTable(const std::string& table_id) { size += engine->PhysicalSize(); if (size > available_size) { - break; + return Status(SERVER_CACHE_FULL, "Cache is full"); } else { try { // step 1: load index @@ -639,8 +639,9 @@ DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const m ENGINE_LOG_DEBUG << "Merging file " << file_schema.file_id_; index_size = index->Size(); - if (index_size >= file_schema.index_file_size_) + if (index_size >= file_schema.index_file_size_) { break; + } } // step 3: serialize to disk diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 5d6a237c1d..471afff41b 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -770,6 +770,16 @@ Config::GetDBConfigStrBuildIndexGPU() { return value; } +std::string +Config::GetDBConfigStrPreloadTable() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE, value).ok()) { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PRELOAD_TABLE); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE, value); + } + return value; +} + //////////////////////////////////////////////////////////////////////////////// /* metric config */ std::string @@ -988,6 +998,12 @@ Config::GetDBConfigBuildIndexGPU(int32_t& value) { return Status::OK(); } +Status +Config::GetDBConfigPreloadTable(std::string& value) { + value = GetDBConfigStrPreloadTable(); + return Status::OK(); +} + Status Config::GetMetricConfigEnableMonitor(bool& value) { std::string str = GetMetricConfigStrEnableMonitor(); diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 9f6932d267..6d9aee37a1 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -56,6 +56,7 @@ static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4"; static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; +static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table"; /* cache config */ static const char* CONFIG_CACHE = "cache_config"; @@ -204,6 +205,8 @@ class Config { GetDBConfigStrInsertBufferSize(); std::string GetDBConfigStrBuildIndexGPU(); + std::string + GetDBConfigStrPreloadTable(); /* metric config */ std::string @@ -261,6 +264,8 @@ class Config { GetDBConfigInsertBufferSize(int32_t& value); Status GetDBConfigBuildIndexGPU(int32_t& value); + Status + GetDBConfigPreloadTable(std::string& value); /* metric config */ Status diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 401c494575..99bf57b3e0 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -155,6 +155,20 @@ DBWrapper::StartService() { db_->Start(); + // preload table + std::string preload_tables; + s = config.GetDBConfigPreloadTable(preload_tables); + if (!s.ok()) { + return s; + } + + s = PreloadTables(preload_tables); + if (!s.ok()) { + std::cerr << "ERROR! Failed to preload tables: " << preload_tables << std::endl; + std::cerr << s.ToString() << std::endl; + kill(0, SIGUSR1); + } + return Status::OK(); } @@ -167,5 +181,34 @@ DBWrapper::StopService() { return Status::OK(); } +Status +DBWrapper::PreloadTables(const std::string& preload_tables) { + if(preload_tables.empty()) { + //do nothing + } else if(preload_tables == "*") { + //load all tables + std::vector table_schema_array; + db_->AllTables(table_schema_array); + + for(auto& schema : table_schema_array) { + auto status = db_->PreloadTable(schema.table_id_); + if (!status.ok()) { + return status; + } + } + } else { + std::vector table_names; + StringHelpFunctions::SplitStringByDelimeter(preload_tables, ",", table_names); + for(auto& name : table_names) { + auto status = db_->PreloadTable(name); + if (!status.ok()) { + return status; + } + } + } + + return Status::OK(); +} + } // namespace server } // namespace milvus diff --git a/cpp/src/server/DBWrapper.h b/cpp/src/server/DBWrapper.h index 08e07c09f6..3508efa63d 100644 --- a/cpp/src/server/DBWrapper.h +++ b/cpp/src/server/DBWrapper.h @@ -52,6 +52,10 @@ class DBWrapper { return db_; } + private: + Status + PreloadTables(const std::string& preload_tables); + private: engine::DBPtr db_; }; diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp index b2aefe4f65..ac35f82947 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp @@ -36,7 +36,6 @@ ErrorMap(ErrorCode code) { {SERVER_INVALID_ARGUMENT, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, {SERVER_FILE_NOT_FOUND, ::milvus::grpc::ErrorCode::FILE_NOT_FOUND}, {SERVER_NOT_IMPLEMENT, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, - {SERVER_BLOCKING_QUEUE_EMPTY, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, {SERVER_CANNOT_CREATE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FOLDER}, {SERVER_CANNOT_CREATE_FILE, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FILE}, {SERVER_CANNOT_DELETE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_DELETE_FOLDER}, @@ -57,7 +56,7 @@ ErrorMap(ErrorCode code) { {SERVER_INVALID_INDEX_FILE_SIZE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, {SERVER_ILLEGAL_VECTOR_ID, ::milvus::grpc::ErrorCode::ILLEGAL_VECTOR_ID}, {SERVER_ILLEGAL_SEARCH_RESULT, ::milvus::grpc::ErrorCode::ILLEGAL_SEARCH_RESULT}, - {SERVER_CACHE_ERROR, ::milvus::grpc::ErrorCode::CACHE_FAILED}, + {SERVER_CACHE_FULL, ::milvus::grpc::ErrorCode::CACHE_FAILED}, {DB_META_TRANSACTION_FAILED, ::milvus::grpc::ErrorCode::META_FAILED}, {SERVER_BUILD_INDEX_ERROR, ::milvus::grpc::ErrorCode::BUILD_INDEX_ERROR}, {SERVER_OUT_OF_MEMORY, ::milvus::grpc::ErrorCode::OUT_OF_MEMORY}, diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index 9cba18ef41..dfc400ca9a 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -56,7 +56,6 @@ constexpr ErrorCode SERVER_NULL_POINTER = ToServerErrorCode(3); constexpr ErrorCode SERVER_INVALID_ARGUMENT = ToServerErrorCode(4); constexpr ErrorCode SERVER_FILE_NOT_FOUND = ToServerErrorCode(5); constexpr ErrorCode SERVER_NOT_IMPLEMENT = ToServerErrorCode(6); -constexpr ErrorCode SERVER_BLOCKING_QUEUE_EMPTY = ToServerErrorCode(7); constexpr ErrorCode SERVER_CANNOT_CREATE_FOLDER = ToServerErrorCode(8); constexpr ErrorCode SERVER_CANNOT_CREATE_FILE = ToServerErrorCode(9); constexpr ErrorCode SERVER_CANNOT_DELETE_FOLDER = ToServerErrorCode(10); @@ -74,7 +73,7 @@ constexpr ErrorCode SERVER_INVALID_ROWRECORD_ARRAY = ToServerErrorCode(107); constexpr ErrorCode SERVER_INVALID_TOPK = ToServerErrorCode(108); constexpr ErrorCode SERVER_ILLEGAL_VECTOR_ID = ToServerErrorCode(109); constexpr ErrorCode SERVER_ILLEGAL_SEARCH_RESULT = ToServerErrorCode(110); -constexpr ErrorCode SERVER_CACHE_ERROR = ToServerErrorCode(111); +constexpr ErrorCode SERVER_CACHE_FULL = ToServerErrorCode(111); constexpr ErrorCode SERVER_WRITE_ERROR = ToServerErrorCode(112); constexpr ErrorCode SERVER_INVALID_NPROBE = ToServerErrorCode(113); constexpr ErrorCode SERVER_INVALID_INDEX_NLIST = ToServerErrorCode(114); From 47b2b1640bede72e8b830c87fa340ade6893523d Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 9 Oct 2019 12:19:42 +0800 Subject: [PATCH 03/12] refine code Former-commit-id: 96432d46a2797134363ae17529d86c5ad6bf148d --- cpp/src/server/Config.cpp | 309 +++++--------------------------------- cpp/src/server/Config.h | 58 +------ 2 files changed, 40 insertions(+), 327 deletions(-) diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 471afff41b..3a88c7f8ba 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -655,304 +655,62 @@ Config::SetConfigValueInMem(const std::string& parent_key, const std::string& ch } //////////////////////////////////////////////////////////////////////////////// -/* server config */ std::string -Config::GetServerConfigStrAddress() { +Config::GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value) { std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); + if (!GetConfigValueInMem(parent_key, child_key, value).ok()) { + value = GetConfigNode(parent_key).GetValue(child_key, default_value); + SetConfigValueInMem(parent_key, child_key, value); } return value; } -std::string -Config::GetServerConfigStrPort() { - std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); - } - return value; -} - -std::string -Config::GetServerConfigStrDeployMode() { - std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_DEPLOY_MODE, CONFIG_SERVER_DEPLOY_MODE_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); - } - return value; -} - -std::string -Config::GetServerConfigStrTimeZone() { - std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); - } - return value; -} - -//////////////////////////////////////////////////////////////////////////////// -/* db config */ -std::string -Config::GetDBConfigStrPrimaryPath() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PRIMARY_PATH, CONFIG_DB_PRIMARY_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); - } - return value; -} - -std::string -Config::GetDBConfigStrSecondaryPath() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SECONDARY_PATH, CONFIG_DB_SECONDARY_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); - } - return value; -} - -std::string -Config::GetDBConfigStrBackendUrl() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL, CONFIG_DB_BACKEND_URL_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); - } - return value; -} - -std::string -Config::GetDBConfigStrArchiveDiskThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, - CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); - } - return value; -} - -std::string -Config::GetDBConfigStrArchiveDaysThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, - CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); - } - return value; -} - -std::string -Config::GetDBConfigStrInsertBufferSize() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); - } - return value; -} - -std::string -Config::GetDBConfigStrBuildIndexGPU() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); - } - return value; -} - -std::string -Config::GetDBConfigStrPreloadTable() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PRELOAD_TABLE); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE, value); - } - return value; -} - -//////////////////////////////////////////////////////////////////////////////// -/* metric config */ -std::string -Config::GetMetricConfigStrEnableMonitor() { - std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value).ok()) { - value = - GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_ENABLE_MONITOR, CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); - } - return value; -} - -std::string -Config::GetMetricConfigStrCollector() { - std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { - value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); - } - return value; -} - -std::string -Config::GetMetricConfigStrPrometheusPort() { - std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { - value = - GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); - } - return value; -} - -//////////////////////////////////////////////////////////////////////////////// -/* cache config */ -std::string -Config::GetCacheConfigStrCpuCacheCapacity() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value).ok()) { - value = GetConfigNode(CONFIG_CACHE) - .GetValue(CONFIG_CACHE_CPU_CACHE_CAPACITY, CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); - } - return value; -} - -std::string -Config::GetCacheConfigStrCpuCacheThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_CACHE) - .GetValue(CONFIG_CACHE_CPU_CACHE_THRESHOLD, CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); - } - return value; -} - -std::string -Config::GetCacheConfigStrGpuCacheCapacity() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value).ok()) { - value = GetConfigNode(CONFIG_CACHE) - .GetValue(CONFIG_CACHE_GPU_CACHE_CAPACITY, CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); - } - return value; -} - -std::string -Config::GetCacheConfigStrGpuCacheThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_CACHE) - .GetValue(CONFIG_CACHE_GPU_CACHE_THRESHOLD, CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); - } - return value; -} - -std::string -Config::GetCacheConfigStrCacheInsertData() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { - value = GetConfigNode(CONFIG_CACHE) - .GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); - } - return value; -} - -//////////////////////////////////////////////////////////////////////////////// -/* engine config */ -std::string -Config::GetEngineConfigStrUseBlasThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_ENGINE) - .GetValue(CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); - } - return value; -} - -std::string -Config::GetEngineConfigStrOmpThreadNum() { - std::string value; - if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { - value = - GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); - } - return value; -} - -//////////////////////////////////////////////////////////////////////////////// -/* resource config */ -std::string -Config::GetResourceConfigStrMode() { - std::string value; - if (!GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { - value = GetConfigNode(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); - } - return value; -} - -//////////////////////////////////////////////////////////////////////////////// Status Config::GetServerConfigAddress(std::string& value) { - value = GetServerConfigStrAddress(); + value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); return CheckServerConfigAddress(value); } Status Config::GetServerConfigPort(std::string& value) { - value = GetServerConfigStrPort(); + value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); return CheckServerConfigPort(value); } Status Config::GetServerConfigDeployMode(std::string& value) { - value = GetServerConfigStrDeployMode(); + value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, CONFIG_SERVER_DEPLOY_MODE_DEFAULT); return CheckServerConfigDeployMode(value); } Status Config::GetServerConfigTimeZone(std::string& value) { - value = GetServerConfigStrTimeZone(); + value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); return CheckServerConfigTimeZone(value); } Status Config::GetDBConfigPrimaryPath(std::string& value) { - value = GetDBConfigStrPrimaryPath(); + value = GetConfigStr(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, CONFIG_DB_PRIMARY_PATH_DEFAULT); return CheckDBConfigPrimaryPath(value); } Status Config::GetDBConfigSecondaryPath(std::string& value) { - value = GetDBConfigStrSecondaryPath(); + value = GetConfigStr(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, CONFIG_DB_SECONDARY_PATH_DEFAULT); return Status::OK(); } Status Config::GetDBConfigBackendUrl(std::string& value) { - value = GetDBConfigStrBackendUrl(); + value = GetConfigStr(CONFIG_DB, CONFIG_DB_BACKEND_URL, CONFIG_DB_BACKEND_URL_DEFAULT); return CheckDBConfigBackendUrl(value); } Status Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { - std::string str = GetDBConfigStrArchiveDiskThreshold(); + std::string str = + GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) { return s; @@ -964,7 +722,8 @@ Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { Status Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { - std::string str = GetDBConfigStrArchiveDaysThreshold(); + std::string str = + GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) { return s; @@ -976,7 +735,8 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { Status Config::GetDBConfigInsertBufferSize(int32_t& value) { - std::string str = GetDBConfigStrInsertBufferSize(); + std::string str = + GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) { return s; @@ -988,7 +748,8 @@ Config::GetDBConfigInsertBufferSize(int32_t& value) { Status Config::GetDBConfigBuildIndexGPU(int32_t& value) { - std::string str = GetDBConfigStrBuildIndexGPU(); + std::string str = + GetConfigStr(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); Status s = CheckDBConfigBuildIndexGPU(str); if (!s.ok()) { return s; @@ -1000,13 +761,14 @@ Config::GetDBConfigBuildIndexGPU(int32_t& value) { Status Config::GetDBConfigPreloadTable(std::string& value) { - value = GetDBConfigStrPreloadTable(); + value = GetConfigStr(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE); return Status::OK(); } Status Config::GetMetricConfigEnableMonitor(bool& value) { - std::string str = GetMetricConfigStrEnableMonitor(); + std::string str = + GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); Status s = CheckMetricConfigEnableMonitor(str); if (!s.ok()) { return s; @@ -1019,19 +781,20 @@ Config::GetMetricConfigEnableMonitor(bool& value) { Status Config::GetMetricConfigCollector(std::string& value) { - value = GetMetricConfigStrCollector(); + value = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); return Status::OK(); } Status Config::GetMetricConfigPrometheusPort(std::string& value) { - value = GetMetricConfigStrPrometheusPort(); + value = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); return CheckMetricConfigPrometheusPort(value); } Status Config::GetCacheConfigCpuCacheCapacity(int32_t& value) { - std::string str = GetCacheConfigStrCpuCacheCapacity(); + std::string str = + GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); Status s = CheckCacheConfigCpuCacheCapacity(str); if (!s.ok()) { return s; @@ -1043,7 +806,8 @@ Config::GetCacheConfigCpuCacheCapacity(int32_t& value) { Status Config::GetCacheConfigCpuCacheThreshold(float& value) { - std::string str = GetCacheConfigStrCpuCacheThreshold(); + std::string str = + GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT); Status s = CheckCacheConfigCpuCacheThreshold(str); if (!s.ok()) { return s; @@ -1055,7 +819,8 @@ Config::GetCacheConfigCpuCacheThreshold(float& value) { Status Config::GetCacheConfigGpuCacheCapacity(int32_t& value) { - std::string str = GetCacheConfigStrGpuCacheCapacity(); + std::string str = + GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); Status s = CheckCacheConfigGpuCacheCapacity(str); if (!s.ok()) { return s; @@ -1067,7 +832,8 @@ Config::GetCacheConfigGpuCacheCapacity(int32_t& value) { Status Config::GetCacheConfigGpuCacheThreshold(float& value) { - std::string str = GetCacheConfigStrGpuCacheThreshold(); + std::string str = + GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); Status s = CheckCacheConfigGpuCacheThreshold(str); if (!s.ok()) { return s; @@ -1079,7 +845,8 @@ Config::GetCacheConfigGpuCacheThreshold(float& value) { Status Config::GetCacheConfigCacheInsertData(bool& value) { - std::string str = GetCacheConfigStrCacheInsertData(); + std::string str = + GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); Status s = CheckCacheConfigCacheInsertData(str); if (!s.ok()) { return s; @@ -1092,7 +859,8 @@ Config::GetCacheConfigCacheInsertData(bool& value) { Status Config::GetEngineConfigUseBlasThreshold(int32_t& value) { - std::string str = GetEngineConfigStrUseBlasThreshold(); + std::string str = + GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); Status s = CheckEngineConfigUseBlasThreshold(str); if (!s.ok()) { return s; @@ -1104,7 +872,8 @@ Config::GetEngineConfigUseBlasThreshold(int32_t& value) { Status Config::GetEngineConfigOmpThreadNum(int32_t& value) { - std::string str = GetEngineConfigStrOmpThreadNum(); + std::string str = + GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) { return s; @@ -1116,7 +885,7 @@ Config::GetEngineConfigOmpThreadNum(int32_t& value) { Status Config::GetResourceConfigMode(std::string& value) { - value = GetResourceConfigStrMode(); + value = GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); return CheckResourceConfigMode(value); } diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 6d9aee37a1..fb00498b96 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -179,64 +179,8 @@ class Config { Status CheckResourceConfigPool(const std::vector& value); - /////////////////////////////////////////////////////////////////////////// - /* server config */ std::string - GetServerConfigStrAddress(); - std::string - GetServerConfigStrPort(); - std::string - GetServerConfigStrDeployMode(); - std::string - GetServerConfigStrTimeZone(); - - /* db config */ - std::string - GetDBConfigStrPrimaryPath(); - std::string - GetDBConfigStrSecondaryPath(); - std::string - GetDBConfigStrBackendUrl(); - std::string - GetDBConfigStrArchiveDiskThreshold(); - std::string - GetDBConfigStrArchiveDaysThreshold(); - std::string - GetDBConfigStrInsertBufferSize(); - std::string - GetDBConfigStrBuildIndexGPU(); - std::string - GetDBConfigStrPreloadTable(); - - /* metric config */ - std::string - GetMetricConfigStrEnableMonitor(); - std::string - GetMetricConfigStrCollector(); - std::string - GetMetricConfigStrPrometheusPort(); - - /* cache config */ - std::string - GetCacheConfigStrCpuCacheCapacity(); - std::string - GetCacheConfigStrCpuCacheThreshold(); - std::string - GetCacheConfigStrGpuCacheCapacity(); - std::string - GetCacheConfigStrGpuCacheThreshold(); - std::string - GetCacheConfigStrCacheInsertData(); - - /* engine config */ - std::string - GetEngineConfigStrUseBlasThreshold(); - std::string - GetEngineConfigStrOmpThreadNum(); - - /* resource config */ - std::string - GetResourceConfigStrMode(); + GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); public: /* server config */ From 438643c9b2d59c6eae0ef170f588e4eed56dad80 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 9 Oct 2019 13:57:49 +0800 Subject: [PATCH 04/12] refine code Former-commit-id: 1aae68fe22fe47062f1c4b1277f4bae0f6e924ec --- cpp/coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/coverage.sh b/cpp/coverage.sh index c0defe6f5a..8a7e5f52a1 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -104,7 +104,7 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ "src/metrics/MetricBase.h"\ "src/server/Server.cpp"\ "src/server/DBWrapper.cpp"\ - "src/server/grpc_impl/GrpcMilvusServer.cpp"\ + "src/server/grpc_impl/GrpcServer.cpp"\ "src/utils/easylogging++.h"\ "src/utils/easylogging++.cc"\ From 406fc339cc4d3c02de6158fb1fd67fca90f36d8a Mon Sep 17 00:00:00 2001 From: test Date: Wed, 9 Oct 2019 06:00:30 +0000 Subject: [PATCH 05/12] MS-575 Add Clang-format & Clang-tidy & Cpplint Former-commit-id: 1a941a50fcdeaa0193be5c6d0031781bc3775d1e --- cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp | 2 +- cpp/src/core/knowhere/knowhere/common/Buffer.h | 2 +- cpp/src/core/unittest/test_ivf.cpp | 2 +- cpp/src/core/unittest/test_kdt.cpp | 2 +- cpp/src/core/unittest/test_nsg/test_nsg.cpp | 2 +- cpp/src/db/insert/MemTableFile.cpp | 6 +++--- cpp/src/scheduler/optimizer/HybridPass.cpp | 6 +++--- cpp/src/scheduler/optimizer/HybridPass.h | 4 ++-- cpp/src/scheduler/optimizer/Optimizer.cpp | 6 +++--- cpp/src/scheduler/optimizer/Optimizer.h | 4 ++-- cpp/src/scheduler/optimizer/Pass.h | 4 ++-- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp index f4b1a7953f..b4c3910a01 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp @@ -76,7 +76,7 @@ ConvertToDataset(std::vector query_results) { auto p_id = (int64_t*)malloc(sizeof(int64_t) * elems); auto p_dist = (float*)malloc(sizeof(float) * elems); -// TODO: throw if malloc failed. + // TODO: throw if malloc failed. #pragma omp parallel for for (auto i = 0; i < query_results.size(); ++i) { diff --git a/cpp/src/core/knowhere/knowhere/common/Buffer.h b/cpp/src/core/knowhere/knowhere/common/Buffer.h index d95da18602..f9e15d95bd 100644 --- a/cpp/src/core/knowhere/knowhere/common/Buffer.h +++ b/cpp/src/core/knowhere/knowhere/common/Buffer.h @@ -36,7 +36,7 @@ struct BufferDeleter { free((void*)buffer->data()); } }; -} +} // namespace internal inline BufferPtr MakeBufferSmart(uint8_t* data, const int64_t size) { diff --git a/cpp/src/core/unittest/test_ivf.cpp b/cpp/src/core/unittest/test_ivf.cpp index a0da0eca8e..17eb888ddc 100644 --- a/cpp/src/core/unittest/test_ivf.cpp +++ b/cpp/src/core/unittest/test_ivf.cpp @@ -43,9 +43,9 @@ namespace kn = knowhere; } // namespace +using ::testing::Combine; using ::testing::TestWithParam; using ::testing::Values; -using ::testing::Combine; constexpr int device_id = 0; constexpr int64_t DIM = 128; diff --git a/cpp/src/core/unittest/test_kdt.cpp b/cpp/src/core/unittest/test_kdt.cpp index 6fa6ba33f3..f9e02bd9a4 100644 --- a/cpp/src/core/unittest/test_kdt.cpp +++ b/cpp/src/core/unittest/test_kdt.cpp @@ -34,9 +34,9 @@ namespace kn = knowhere; } // namespace +using ::testing::Combine; using ::testing::TestWithParam; using ::testing::Values; -using ::testing::Combine; class KDTTest : public DataGen, public ::testing::Test { protected: diff --git a/cpp/src/core/unittest/test_nsg/test_nsg.cpp b/cpp/src/core/unittest/test_nsg/test_nsg.cpp index 70261a63dc..b59f0d4928 100644 --- a/cpp/src/core/unittest/test_nsg/test_nsg.cpp +++ b/cpp/src/core/unittest/test_nsg/test_nsg.cpp @@ -32,9 +32,9 @@ namespace kn = knowhere; } // namespace +using ::testing::Combine; using ::testing::TestWithParam; using ::testing::Values; -using ::testing::Combine; constexpr int64_t DEVICE_ID = 1; diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 5827513297..2c877c78ed 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -55,9 +55,9 @@ MemTableFile::CreateTableFile() { Status MemTableFile::Add(const VectorSourcePtr& source, IDNumbers& vector_ids) { if (table_file_schema_.dimension_ <= 0) { - std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + - std::to_string(table_file_schema_.dimension_) + ", table_id = " + - table_file_schema_.table_id_; + std::string err_msg = + "MemTableFile::Add: table_file_schema dimension = " + std::to_string(table_file_schema_.dimension_) + + ", table_id = " + table_file_schema_.table_id_; ENGINE_LOG_ERROR << err_msg; return Status(DB_ERROR, "Not able to create table file"); } diff --git a/cpp/src/scheduler/optimizer/HybridPass.cpp b/cpp/src/scheduler/optimizer/HybridPass.cpp index 1b0ab7c4d0..dbfab0f741 100644 --- a/cpp/src/scheduler/optimizer/HybridPass.cpp +++ b/cpp/src/scheduler/optimizer/HybridPass.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "HybridPass.h" +#include "cpp/src/scheduler/optimizer/HybridPass.h" #include "scheduler/task/SearchTask.h" namespace milvus { @@ -30,6 +30,6 @@ HybridPass::Run(const TaskPtr& task) { return false; } -} -} +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/HybridPass.h b/cpp/src/scheduler/optimizer/HybridPass.h index e043bf8ad5..1d1d1e227f 100644 --- a/cpp/src/scheduler/optimizer/HybridPass.h +++ b/cpp/src/scheduler/optimizer/HybridPass.h @@ -43,6 +43,6 @@ class HybridPass : public Pass { using HybridPassPtr = std::shared_ptr; -} -} +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/Optimizer.cpp b/cpp/src/scheduler/optimizer/Optimizer.cpp index 517cd85f79..6e02bfd1c8 100644 --- a/cpp/src/scheduler/optimizer/Optimizer.cpp +++ b/cpp/src/scheduler/optimizer/Optimizer.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "Optimizer.h" +#include "cpp/src/scheduler/optimizer/Optimizer.h" namespace milvus { namespace scheduler { @@ -38,6 +38,6 @@ Optimizer::Run(const TaskPtr& task) { return false; } -} -} +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/Optimizer.h b/cpp/src/scheduler/optimizer/Optimizer.h index cbb13a8fc2..1a6e0e5d6b 100644 --- a/cpp/src/scheduler/optimizer/Optimizer.h +++ b/cpp/src/scheduler/optimizer/Optimizer.h @@ -46,6 +46,6 @@ class Optimizer { std::vector pass_list_; }; -} -} +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/Pass.h b/cpp/src/scheduler/optimizer/Pass.h index 1c8def41a2..e63b1b369c 100644 --- a/cpp/src/scheduler/optimizer/Pass.h +++ b/cpp/src/scheduler/optimizer/Pass.h @@ -42,5 +42,5 @@ class Pass { }; using PassPtr = std::shared_ptr; -} -} +} // namespace scheduler +} // namespace milvus From bc23b7a3849ac1676997dc4f1c88f9b765774d29 Mon Sep 17 00:00:00 2001 From: test Date: Wed, 9 Oct 2019 06:10:36 +0000 Subject: [PATCH 06/12] MS-575 Add Clang-format & Clang-tidy & Cpplint Former-commit-id: d20bf96022680c43b050d9d5327ee77dcfa1ba80 --- cpp/src/db/DBImpl.cpp | 3 +- cpp/src/db/meta/MySQLMetaImpl.cpp | 21 +++++----- cpp/src/scheduler/optimizer/HybridPass.cpp | 8 ++-- cpp/src/scheduler/optimizer/HybridPass.h | 21 +++++----- cpp/src/scheduler/optimizer/Optimizer.cpp | 5 +-- cpp/src/scheduler/optimizer/Optimizer.h | 23 ++++++----- cpp/src/scheduler/optimizer/Pass.h | 23 +++++------ cpp/src/scheduler/task/SearchTask.cpp | 4 +- cpp/src/sdk/include/Status.h | 10 ++--- cpp/src/server/Config.cpp | 12 ++---- cpp/src/server/DBWrapper.cpp | 13 ++++--- cpp/src/server/DBWrapper.h | 1 + cpp/src/server/grpc_impl/GrpcRequestHandler.h | 38 +++++++++---------- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 4 +- cpp/src/utils/LogUtil.cpp | 2 +- 15 files changed, 94 insertions(+), 94 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 93f71dc737..729a50a7c3 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -296,7 +296,8 @@ DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { std::vector file_types; if (index.engine_type_ == static_cast(EngineType::FAISS_IDMAP)) { file_types = { - static_cast(meta::TableFileSchema::NEW), static_cast(meta::TableFileSchema::NEW_MERGE), + static_cast(meta::TableFileSchema::NEW), + static_cast(meta::TableFileSchema::NEW_MERGE), }; } else { file_types = { diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 872c1ced2e..f9f1569a65 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -148,15 +148,18 @@ static const MetaSchema TABLES_SCHEMA(META_TABLES, { }); // TableFiles schema -static const MetaSchema TABLEFILES_SCHEMA( - META_TABLEFILES, - { - MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), MetaField("table_id", "VARCHAR(255)", "NOT NULL"), - MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), MetaField("file_id", "VARCHAR(255)", "NOT NULL"), - MetaField("file_type", "INT", "DEFAULT 0 NOT NULL"), MetaField("file_size", "BIGINT", "DEFAULT 0 NOT NULL"), - MetaField("row_count", "BIGINT", "DEFAULT 0 NOT NULL"), MetaField("updated_time", "BIGINT", "NOT NULL"), - MetaField("created_on", "BIGINT", "NOT NULL"), MetaField("date", "INT", "DEFAULT -1 NOT NULL"), - }); +static const MetaSchema TABLEFILES_SCHEMA(META_TABLEFILES, { + MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), + MetaField("table_id", "VARCHAR(255)", "NOT NULL"), + MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), + MetaField("file_id", "VARCHAR(255)", "NOT NULL"), + MetaField("file_type", "INT", "DEFAULT 0 NOT NULL"), + MetaField("file_size", "BIGINT", "DEFAULT 0 NOT NULL"), + MetaField("row_count", "BIGINT", "DEFAULT 0 NOT NULL"), + MetaField("updated_time", "BIGINT", "NOT NULL"), + MetaField("created_on", "BIGINT", "NOT NULL"), + MetaField("date", "INT", "DEFAULT -1 NOT NULL"), + }); } // namespace diff --git a/cpp/src/scheduler/optimizer/HybridPass.cpp b/cpp/src/scheduler/optimizer/HybridPass.cpp index dbfab0f741..85b28d9c30 100644 --- a/cpp/src/scheduler/optimizer/HybridPass.cpp +++ b/cpp/src/scheduler/optimizer/HybridPass.cpp @@ -24,12 +24,12 @@ namespace scheduler { bool HybridPass::Run(const TaskPtr& task) { // TODO: Index::IVFSQ8Hybrid, if nq < threshold set cpu, else set gpu - if (task->Type() != TaskType::SearchTask) return false; + if (task->Type() != TaskType::SearchTask) + return false; auto search_task = std::static_pointer_cast(task); // if (search_task->file_->engine_type_ == engine::EngineType::FAISS_IVFSQ8Hybrid) return false; } -} // namespace scheduler -} // namespace milvus - +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/HybridPass.h b/cpp/src/scheduler/optimizer/HybridPass.h index 1d1d1e227f..0d02a8bda9 100644 --- a/cpp/src/scheduler/optimizer/HybridPass.h +++ b/cpp/src/scheduler/optimizer/HybridPass.h @@ -16,16 +16,16 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include #include "Pass.h" @@ -43,6 +43,5 @@ class HybridPass : public Pass { using HybridPassPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus - +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/Optimizer.cpp b/cpp/src/scheduler/optimizer/Optimizer.cpp index 6e02bfd1c8..d50bd05899 100644 --- a/cpp/src/scheduler/optimizer/Optimizer.cpp +++ b/cpp/src/scheduler/optimizer/Optimizer.cpp @@ -38,6 +38,5 @@ Optimizer::Run(const TaskPtr& task) { return false; } -} // namespace scheduler -} // namespace milvus - +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/Optimizer.h b/cpp/src/scheduler/optimizer/Optimizer.h index 1a6e0e5d6b..99282e66a6 100644 --- a/cpp/src/scheduler/optimizer/Optimizer.h +++ b/cpp/src/scheduler/optimizer/Optimizer.h @@ -16,16 +16,16 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include #include "Pass.h" @@ -40,12 +40,11 @@ class Optimizer { Init(); bool - Run(const TaskPtr &task); + Run(const TaskPtr& task); private: std::vector pass_list_; }; -} // namespace scheduler -} // namespace milvus - +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/Pass.h b/cpp/src/scheduler/optimizer/Pass.h index e63b1b369c..959c3ea5ee 100644 --- a/cpp/src/scheduler/optimizer/Pass.h +++ b/cpp/src/scheduler/optimizer/Pass.h @@ -16,16 +16,16 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include #include "scheduler/task/Task.h" @@ -35,12 +35,13 @@ namespace scheduler { class Pass { public: virtual void - Init() {} + Init() { + } virtual bool Run(const TaskPtr& task) = 0; }; using PassPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/task/SearchTask.cpp b/cpp/src/scheduler/task/SearchTask.cpp index ee24dace43..9925a8bcf8 100644 --- a/cpp/src/scheduler/task/SearchTask.cpp +++ b/cpp/src/scheduler/task/SearchTask.cpp @@ -157,8 +157,8 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { size_t file_size = index_engine_->PhysicalSize(); - std::string info = "Load file id:" + std::to_string(file_->id_) + " file type:" + - std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + + std::string info = "Load file id:" + std::to_string(file_->id_) + + " file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + " bytes from location: " + file_->location_ + " totally cost"; double span = rc.ElapseFromBegin(info); // for (auto &context : search_contexts_) { diff --git a/cpp/src/sdk/include/Status.h b/cpp/src/sdk/include/Status.h index a81116b31d..008f9956d2 100644 --- a/cpp/src/sdk/include/Status.h +++ b/cpp/src/sdk/include/Status.h @@ -20,12 +20,12 @@ #include /** \brief Milvus SDK namespace -*/ + */ namespace milvus { /** -* @brief Status Code for SDK interface return -*/ + * @brief Status Code for SDK interface return + */ enum class StatusCode { OK = 0, @@ -41,8 +41,8 @@ enum class StatusCode { }; /** -* @brief Status for SDK interface return -*/ + * @brief Status for SDK interface return + */ class Status { public: Status(StatusCode code, const std::string& msg); diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 3a88c7f8ba..d383eb5edd 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -735,8 +735,7 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { Status Config::GetDBConfigInsertBufferSize(int32_t& value) { - std::string str = - GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); + std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) { return s; @@ -748,8 +747,7 @@ Config::GetDBConfigInsertBufferSize(int32_t& value) { Status Config::GetDBConfigBuildIndexGPU(int32_t& value) { - std::string str = - GetConfigStr(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); + std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); Status s = CheckDBConfigBuildIndexGPU(str); if (!s.ok()) { return s; @@ -767,8 +765,7 @@ Config::GetDBConfigPreloadTable(std::string& value) { Status Config::GetMetricConfigEnableMonitor(bool& value) { - std::string str = - GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); + std::string str = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); Status s = CheckMetricConfigEnableMonitor(str); if (!s.ok()) { return s; @@ -872,8 +869,7 @@ Config::GetEngineConfigUseBlasThreshold(int32_t& value) { Status Config::GetEngineConfigOmpThreadNum(int32_t& value) { - std::string str = - GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); + std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) { return s; diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 99bf57b3e0..306863f8ae 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -25,6 +25,7 @@ #include #include #include +#include namespace milvus { namespace server { @@ -183,14 +184,14 @@ DBWrapper::StopService() { Status DBWrapper::PreloadTables(const std::string& preload_tables) { - if(preload_tables.empty()) { - //do nothing - } else if(preload_tables == "*") { - //load all tables + if (preload_tables.empty()) { + // do nothing + } else if (preload_tables == "*") { + // load all tables std::vector table_schema_array; db_->AllTables(table_schema_array); - for(auto& schema : table_schema_array) { + for (auto& schema : table_schema_array) { auto status = db_->PreloadTable(schema.table_id_); if (!status.ok()) { return status; @@ -199,7 +200,7 @@ DBWrapper::PreloadTables(const std::string& preload_tables) { } else { std::vector table_names; StringHelpFunctions::SplitStringByDelimeter(preload_tables, ",", table_names); - for(auto& name : table_names) { + for (auto& name : table_names) { auto status = db_->PreloadTable(name); if (!status.ok()) { return status; diff --git a/cpp/src/server/DBWrapper.h b/cpp/src/server/DBWrapper.h index 3508efa63d..7016aa8805 100644 --- a/cpp/src/server/DBWrapper.h +++ b/cpp/src/server/DBWrapper.h @@ -21,6 +21,7 @@ #include "utils/Status.h" #include +#include namespace milvus { namespace server { diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.h b/cpp/src/server/grpc_impl/GrpcRequestHandler.h index 0decf61500..1a9b591154 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.h @@ -148,25 +148,25 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { ::milvus::grpc::TopKQueryResultList* response) override; /** - * @brief Internal use query interface - * - * This method is used to query vector in specified files. - * - * @param context, add context for every RPC - * @param request: - * file_id_array, specified files id array, queried. - * query_record_array, all vector are going to be queried. - * query_range_array, optional ranges for conditional search. If not specified, search whole table - * topk, how many similarity vectors will be searched. - * - * @param writer, write query result array. - * - * @return status - * - * @param context - * @param request - * @param writer - */ + * @brief Internal use query interface + * + * This method is used to query vector in specified files. + * + * @param context, add context for every RPC + * @param request: + * file_id_array, specified files id array, queried. + * query_record_array, all vector are going to be queried. + * query_range_array, optional ranges for conditional search. If not specified, search whole table + * topk, how many similarity vectors will be searched. + * + * @param writer, write query result array. + * + * @return status + * + * @param context + * @param request + * @param writer + */ ::grpc::Status SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResultList* response) override; diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index 9f0b8fd95d..dc2bd340eb 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -511,8 +511,8 @@ InsertTask::OnExecute() { } // step 6: update table flag - user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID : table_info.flag_ |= - engine::meta::FLAG_MASK_NO_USERID; + user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID + : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; status = DBWrapper::DB()->UpdateTableFlag(insert_param_->table_name(), table_info.flag_); #ifdef MILVUS_ENABLE_PROFILING diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index ed2e1a446b..4a962f466c 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -31,7 +31,7 @@ static int warning_idx = 0; static int trace_idx = 0; static int error_idx = 0; static int fatal_idx = 0; -} +} // namespace // TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename void From 2b1240e250b6d340fca30a334755e41ceba2635e Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 9 Oct 2019 14:17:17 +0800 Subject: [PATCH 07/12] refine code Former-commit-id: dea5807b12170595c45d402802b5a88fecf24b57 --- cpp/src/scheduler/optimizer/HybridPass.cpp | 10 +++++----- cpp/src/scheduler/optimizer/HybridPass.h | 21 ++++++++++---------- cpp/src/scheduler/optimizer/Optimizer.cpp | 7 +++---- cpp/src/scheduler/optimizer/Optimizer.h | 23 +++++++++++----------- cpp/src/scheduler/optimizer/Pass.h | 23 +++++++++++----------- cpp/src/server/Config.cpp | 12 ++++------- cpp/src/server/DBWrapper.cpp | 13 ++++++------ cpp/src/server/DBWrapper.h | 1 + 8 files changed, 53 insertions(+), 57 deletions(-) diff --git a/cpp/src/scheduler/optimizer/HybridPass.cpp b/cpp/src/scheduler/optimizer/HybridPass.cpp index 1b0ab7c4d0..f172a7beb9 100644 --- a/cpp/src/scheduler/optimizer/HybridPass.cpp +++ b/cpp/src/scheduler/optimizer/HybridPass.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "HybridPass.h" +#include "scheduler/optimizer/HybridPass.h" #include "scheduler/task/SearchTask.h" namespace milvus { @@ -24,12 +24,12 @@ namespace scheduler { bool HybridPass::Run(const TaskPtr& task) { // TODO: Index::IVFSQ8Hybrid, if nq < threshold set cpu, else set gpu - if (task->Type() != TaskType::SearchTask) return false; + if (task->Type() != TaskType::SearchTask) + return false; auto search_task = std::static_pointer_cast(task); // if (search_task->file_->engine_type_ == engine::EngineType::FAISS_IVFSQ8Hybrid) return false; } -} -} - +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/HybridPass.h b/cpp/src/scheduler/optimizer/HybridPass.h index e043bf8ad5..0d02a8bda9 100644 --- a/cpp/src/scheduler/optimizer/HybridPass.h +++ b/cpp/src/scheduler/optimizer/HybridPass.h @@ -16,16 +16,16 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include #include "Pass.h" @@ -43,6 +43,5 @@ class HybridPass : public Pass { using HybridPassPtr = std::shared_ptr; -} -} - +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/Optimizer.cpp b/cpp/src/scheduler/optimizer/Optimizer.cpp index 517cd85f79..c5fa311a27 100644 --- a/cpp/src/scheduler/optimizer/Optimizer.cpp +++ b/cpp/src/scheduler/optimizer/Optimizer.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "Optimizer.h" +#include "scheduler/optimizer/Optimizer.h" namespace milvus { namespace scheduler { @@ -38,6 +38,5 @@ Optimizer::Run(const TaskPtr& task) { return false; } -} -} - +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/Optimizer.h b/cpp/src/scheduler/optimizer/Optimizer.h index cbb13a8fc2..99282e66a6 100644 --- a/cpp/src/scheduler/optimizer/Optimizer.h +++ b/cpp/src/scheduler/optimizer/Optimizer.h @@ -16,16 +16,16 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include #include "Pass.h" @@ -40,12 +40,11 @@ class Optimizer { Init(); bool - Run(const TaskPtr &task); + Run(const TaskPtr& task); private: std::vector pass_list_; }; -} -} - +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/optimizer/Pass.h b/cpp/src/scheduler/optimizer/Pass.h index 1c8def41a2..959c3ea5ee 100644 --- a/cpp/src/scheduler/optimizer/Pass.h +++ b/cpp/src/scheduler/optimizer/Pass.h @@ -16,16 +16,16 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include #include "scheduler/task/Task.h" @@ -35,12 +35,13 @@ namespace scheduler { class Pass { public: virtual void - Init() {} + Init() { + } virtual bool Run(const TaskPtr& task) = 0; }; using PassPtr = std::shared_ptr; -} -} +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 3a88c7f8ba..d383eb5edd 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -735,8 +735,7 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { Status Config::GetDBConfigInsertBufferSize(int32_t& value) { - std::string str = - GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); + std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) { return s; @@ -748,8 +747,7 @@ Config::GetDBConfigInsertBufferSize(int32_t& value) { Status Config::GetDBConfigBuildIndexGPU(int32_t& value) { - std::string str = - GetConfigStr(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); + std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); Status s = CheckDBConfigBuildIndexGPU(str); if (!s.ok()) { return s; @@ -767,8 +765,7 @@ Config::GetDBConfigPreloadTable(std::string& value) { Status Config::GetMetricConfigEnableMonitor(bool& value) { - std::string str = - GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); + std::string str = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); Status s = CheckMetricConfigEnableMonitor(str); if (!s.ok()) { return s; @@ -872,8 +869,7 @@ Config::GetEngineConfigUseBlasThreshold(int32_t& value) { Status Config::GetEngineConfigOmpThreadNum(int32_t& value) { - std::string str = - GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); + std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) { return s; diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 99bf57b3e0..306863f8ae 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -25,6 +25,7 @@ #include #include #include +#include namespace milvus { namespace server { @@ -183,14 +184,14 @@ DBWrapper::StopService() { Status DBWrapper::PreloadTables(const std::string& preload_tables) { - if(preload_tables.empty()) { - //do nothing - } else if(preload_tables == "*") { - //load all tables + if (preload_tables.empty()) { + // do nothing + } else if (preload_tables == "*") { + // load all tables std::vector table_schema_array; db_->AllTables(table_schema_array); - for(auto& schema : table_schema_array) { + for (auto& schema : table_schema_array) { auto status = db_->PreloadTable(schema.table_id_); if (!status.ok()) { return status; @@ -199,7 +200,7 @@ DBWrapper::PreloadTables(const std::string& preload_tables) { } else { std::vector table_names; StringHelpFunctions::SplitStringByDelimeter(preload_tables, ",", table_names); - for(auto& name : table_names) { + for (auto& name : table_names) { auto status = db_->PreloadTable(name); if (!status.ok()) { return status; diff --git a/cpp/src/server/DBWrapper.h b/cpp/src/server/DBWrapper.h index 3508efa63d..7016aa8805 100644 --- a/cpp/src/server/DBWrapper.h +++ b/cpp/src/server/DBWrapper.h @@ -21,6 +21,7 @@ #include "utils/Status.h" #include +#include namespace milvus { namespace server { From e4e2faf416627a2c0694c9ce4e25c3cd4ebac47e Mon Sep 17 00:00:00 2001 From: test Date: Wed, 9 Oct 2019 06:18:55 +0000 Subject: [PATCH 08/12] MS-575 Add Clang-format & Clang-tidy & Cpplint Former-commit-id: 0a31f6c2b8b5c36a019c2d4f89cacac33006960c --- .../knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp | 2 +- .../core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp | 2 +- .../core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp index 4fc9a82009..2612a63630 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include #include #include #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/FaissBaseIndex.h" +#include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/helpers/FaissIO.h" namespace knowhere { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp index dc886b7383..d538c0dea1 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -#include #include #include #include @@ -26,6 +25,7 @@ #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexIVFPQ.h" #include "knowhere/index/vector_index/helpers/Cloner.h" #include "knowhere/index/vector_index/helpers/FaissIO.h" diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp index 077f0817da..75f5135df4 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -#include #include #include #include @@ -23,6 +22,7 @@ #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/IndexGPUIVFPQ.h" +#include "knowhere/index/vector_index/IndexIVFPQ.h" namespace knowhere { From 5253ee01fdd052700a7e957e563c10d2dfaf436c Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 9 Oct 2019 14:59:30 +0800 Subject: [PATCH 09/12] refine code Former-commit-id: 92557f67f9034c38da747cb75ad4c1fb8278e645 --- .../knowhere/adapter/SptagAdapter.cpp | 2 +- cpp/src/db/DBImpl.cpp | 3 ++- cpp/src/db/insert/MemTableFile.cpp | 6 +++--- cpp/src/db/meta/MySQLMetaImpl.cpp | 21 +++++++++++-------- cpp/src/scheduler/task/SearchTask.cpp | 4 ++-- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 4 ++-- 6 files changed, 22 insertions(+), 18 deletions(-) diff --git a/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp index f4b1a7953f..b4c3910a01 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp @@ -76,7 +76,7 @@ ConvertToDataset(std::vector query_results) { auto p_id = (int64_t*)malloc(sizeof(int64_t) * elems); auto p_dist = (float*)malloc(sizeof(float) * elems); -// TODO: throw if malloc failed. + // TODO: throw if malloc failed. #pragma omp parallel for for (auto i = 0; i < query_results.size(); ++i) { diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 93f71dc737..729a50a7c3 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -296,7 +296,8 @@ DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { std::vector file_types; if (index.engine_type_ == static_cast(EngineType::FAISS_IDMAP)) { file_types = { - static_cast(meta::TableFileSchema::NEW), static_cast(meta::TableFileSchema::NEW_MERGE), + static_cast(meta::TableFileSchema::NEW), + static_cast(meta::TableFileSchema::NEW_MERGE), }; } else { file_types = { diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 5827513297..2c877c78ed 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -55,9 +55,9 @@ MemTableFile::CreateTableFile() { Status MemTableFile::Add(const VectorSourcePtr& source, IDNumbers& vector_ids) { if (table_file_schema_.dimension_ <= 0) { - std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + - std::to_string(table_file_schema_.dimension_) + ", table_id = " + - table_file_schema_.table_id_; + std::string err_msg = + "MemTableFile::Add: table_file_schema dimension = " + std::to_string(table_file_schema_.dimension_) + + ", table_id = " + table_file_schema_.table_id_; ENGINE_LOG_ERROR << err_msg; return Status(DB_ERROR, "Not able to create table file"); } diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 872c1ced2e..f9f1569a65 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -148,15 +148,18 @@ static const MetaSchema TABLES_SCHEMA(META_TABLES, { }); // TableFiles schema -static const MetaSchema TABLEFILES_SCHEMA( - META_TABLEFILES, - { - MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), MetaField("table_id", "VARCHAR(255)", "NOT NULL"), - MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), MetaField("file_id", "VARCHAR(255)", "NOT NULL"), - MetaField("file_type", "INT", "DEFAULT 0 NOT NULL"), MetaField("file_size", "BIGINT", "DEFAULT 0 NOT NULL"), - MetaField("row_count", "BIGINT", "DEFAULT 0 NOT NULL"), MetaField("updated_time", "BIGINT", "NOT NULL"), - MetaField("created_on", "BIGINT", "NOT NULL"), MetaField("date", "INT", "DEFAULT -1 NOT NULL"), - }); +static const MetaSchema TABLEFILES_SCHEMA(META_TABLEFILES, { + MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), + MetaField("table_id", "VARCHAR(255)", "NOT NULL"), + MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), + MetaField("file_id", "VARCHAR(255)", "NOT NULL"), + MetaField("file_type", "INT", "DEFAULT 0 NOT NULL"), + MetaField("file_size", "BIGINT", "DEFAULT 0 NOT NULL"), + MetaField("row_count", "BIGINT", "DEFAULT 0 NOT NULL"), + MetaField("updated_time", "BIGINT", "NOT NULL"), + MetaField("created_on", "BIGINT", "NOT NULL"), + MetaField("date", "INT", "DEFAULT -1 NOT NULL"), + }); } // namespace diff --git a/cpp/src/scheduler/task/SearchTask.cpp b/cpp/src/scheduler/task/SearchTask.cpp index ee24dace43..9925a8bcf8 100644 --- a/cpp/src/scheduler/task/SearchTask.cpp +++ b/cpp/src/scheduler/task/SearchTask.cpp @@ -157,8 +157,8 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { size_t file_size = index_engine_->PhysicalSize(); - std::string info = "Load file id:" + std::to_string(file_->id_) + " file type:" + - std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + + std::string info = "Load file id:" + std::to_string(file_->id_) + + " file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + " bytes from location: " + file_->location_ + " totally cost"; double span = rc.ElapseFromBegin(info); // for (auto &context : search_contexts_) { diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index 9f0b8fd95d..dc2bd340eb 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -511,8 +511,8 @@ InsertTask::OnExecute() { } // step 6: update table flag - user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID : table_info.flag_ |= - engine::meta::FLAG_MASK_NO_USERID; + user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID + : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; status = DBWrapper::DB()->UpdateTableFlag(insert_param_->table_name(), table_info.flag_); #ifdef MILVUS_ENABLE_PROFILING From 9c664d9bf94ae146fcfced2182c14b99b2f664f5 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 9 Oct 2019 15:04:50 +0800 Subject: [PATCH 10/12] modify readme Former-commit-id: d288d897945c04151691bfac0dce0fdbf4d9c81a --- cpp/README.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index f5f77d25f8..9211a5d876 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -105,13 +105,20 @@ please reinstall CMake with curl: ``` ##### code format and linting - +Install clang-format and clang-tidy ```shell CentOS 7: $ yum install clang -Ubuntu 16.04 or 18.04: -$ sudo apt-get install clang-format clang-tidy - +Ubuntu 16.04: +$ sudo apt-get install clang-tidy +$ wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - +$ sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main" +$ sudo apt-get update +$ sudo apt-get install clang-format-6.0 +Ubuntu 18.04: +$ sudo apt-get install clang-tidy clang-format +``` +```shell $ ./build.sh -l ``` @@ -122,13 +129,14 @@ $ ./build.sh -u ``` ##### Run code coverage - +Install lcov ```shell CentOS 7: $ yum install lcov Ubuntu 16.04 or 18.04: $ sudo apt-get install lcov - +``` +```shell $ ./build.sh -u -c ``` From 98447500c79cbfec58d1af3ac969cba625d66842 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 9 Oct 2019 15:26:04 +0800 Subject: [PATCH 11/12] fix break build Former-commit-id: 588cb86bdc60cd3d155692be5a875ec2a7ea2233 --- .../core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp index 75f5135df4..213141b3ac 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include #include #include #include From c7f093ce6b8bc24408951e8868778710d6944c6c Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Wed, 9 Oct 2019 17:09:12 +0800 Subject: [PATCH 12/12] change perror into log Former-commit-id: b482d6df5aac705fd8b4f6b57ef936678ee40856 --- cpp/CHANGELOG.md | 1 + cpp/src/metrics/SystemInfo.cpp | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 8f3079b8dd..b7cf014f63 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -10,6 +10,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-577 - Unittest Query randomly hung - MS-587 - Count get wrong result after adding vectors and index built immediately - MS-599 - search wrong result when table created with metric_type: IP +- MS-601 - Docker logs error caused by get CPUTemperature error ## Improvement - MS-552 - Add and change the easylogging library diff --git a/cpp/src/metrics/SystemInfo.cpp b/cpp/src/metrics/SystemInfo.cpp index 70e917ad04..a49c3070a6 100644 --- a/cpp/src/metrics/SystemInfo.cpp +++ b/cpp/src/metrics/SystemInfo.cpp @@ -16,6 +16,7 @@ // under the License. #include "metrics/SystemInfo.h" +#include "utils/Log.h" #include #include @@ -63,12 +64,12 @@ SystemInfo::Init() { nvmlReturn_t nvmlresult; nvmlresult = nvmlInit(); if (NVML_SUCCESS != nvmlresult) { - printf("System information initilization failed"); + SERVER_LOG_ERROR << "System information initilization failed"; return; } nvmlresult = nvmlDeviceGetCount(&num_device_); if (NVML_SUCCESS != nvmlresult) { - printf("Unable to get devidce number"); + SERVER_LOG_ERROR << "Unable to get devidce number"; return; } @@ -154,7 +155,7 @@ SystemInfo::getTotalCpuTime(std::vector& work_time_array) { std::vector total_time_array; FILE* file = fopen("/proc/stat", "r"); if (file == NULL) { - perror("Could not open stat file"); + SERVER_LOG_ERROR << "Could not open stat file"; return total_time_array; } @@ -165,7 +166,7 @@ SystemInfo::getTotalCpuTime(std::vector& work_time_array) { char buffer[1024]; char* ret = fgets(buffer, sizeof(buffer) - 1, file); if (ret == NULL) { - perror("Could not read stat file"); + SERVER_LOG_ERROR << "Could not read stat file"; fclose(file); return total_time_array; } @@ -245,7 +246,7 @@ SystemInfo::CPUTemperature() { DIR *dir = NULL; dir = opendir(path.c_str()); if (!dir) { - perror("opendir"); + SERVER_LOG_ERROR << "Could not open hwmon directory"; return result; } @@ -262,7 +263,7 @@ SystemInfo::CPUTemperature() { object += "/temp1_input"; FILE *file = fopen(object.c_str(), "r"); if (file == nullptr) { - perror("Could not open temperature file"); + SERVER_LOG_ERROR << "Could not open temperature file" exit(1); } float temp;