From d0b888c01239fcb183b227d20845fec9340e981c Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 26 Sep 2019 18:49:32 +0800 Subject: [PATCH 1/2] MS-574 refine configuration names Former-commit-id: c8f29009c5dc6712a0fd67185e24d4f9ce1286d1 --- cpp/conf/server_config.template | 40 +++-- cpp/src/metrics/PrometheusMetrics.cpp | 2 +- cpp/src/server/Config.cpp | 218 +++++++++++++++++++------- cpp/src/server/Config.h | 51 +++--- cpp/src/server/DBWrapper.cpp | 4 +- cpp/src/server/Server.cpp | 6 + 6 files changed, 217 insertions(+), 104 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 6364698100..d80dfda958 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -1,39 +1,37 @@ +# All the following configurations are default values. + server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # port range: 1025 ~ 65534 - mode: single # deployment type: single, cluster, read_only + address: 0.0.0.0 # milvus server ip address (IPv4) + port: 19530 # port range: 1025 ~ 65534 + deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable time_zone: UTC+8 db_config: - path: @MILVUS_DB_PATH@ # milvus database path - slave_path: # secondary database path, split by semicolon + primary_path: @MILVUS_DB_PATH@ # path used to store data and meta + secondary_path: # path used to store data only, split by semicolon - # URI format: dialect://username:password@host:port/database - # All parts except dialect are optional, but you MUST include the delimiters - # Currently dialect supports mysql or sqlite - backend_url: sqlite://:@:/ + backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database + # Keep 'dialect://:@:/', and replace other texts with real values. + # Replace 'dialect' with 'mysql' or 'sqlite' - archive_disk_threshold: 0 # GB, file will be archived when disk usage exceed, 0 for no limit - archive_days_threshold: 0 # DAYS, older files will be archived, 0 for no limit - buffer_size: 4 # GB, maximum insert buffer size allowed - build_index_gpu: 0 # gpu id used for building index + insert_buffer_size: 4 # GB, maximum insert buffer size allowed + build_index_gpu: 0 # gpu id used for building index metric_config: - auto_bootup: off # whether enable monitoring when bootup - collector: prometheus # prometheus + enable_monitor: false # enable monitoring or not + collector: prometheus # prometheus prometheus_config: - port: 8080 # port prometheus used to fetch metrics + port: 8080 # port prometheus used to fetch metrics cache_config: - cpu_mem_capacity: 16 # GB, CPU memory size used for cache - cpu_mem_threshold: 0.85 # percent of data kept when cache cleanup triggered - cache_insert_data: false # whether load data into cache when insert + cpu_mem_capacity: 16 # GB, CPU memory used for cache + cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered + cache_insert_data: false # whether load inserted data into cache engine_config: blas_threshold: 20 resource_config: - mode: simple - pool: + resource_pool: - cpu - gpu0 diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index b607dbb020..6f3513072a 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -31,7 +31,7 @@ ErrorCode PrometheusMetrics::Init() { try { Config &config = Config::GetInstance(); - Status s = config.GetMetricConfigAutoBootup(startup_); + Status s = config.GetMetricConfigEnableMonitor(startup_); if (!s.ok()) return s.code(); if (!startup_) return SERVER_SUCCESS; diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 8ee2f7c62b..3730ff6f68 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -70,6 +70,111 @@ Config::LoadConfigFile(const std::string &filename) { return Status::OK(); } +Status +Config::ValidateConfig() { + Status s; + + /* server config */ + std::string server_addr; + s = GetServerConfigAddress(server_addr); + if (!s.ok()) return s; + + std::string server_port; + s = GetServerConfigPort(server_port); + if (!s.ok()) return s; + + std::string server_mode; + s = GetServerConfigMode(server_mode); + if (!s.ok()) return s; + + std::string server_time_zone; + s = GetServerConfigTimeZone(server_time_zone); + if (!s.ok()) return s; + + /* db config */ + std::string db_primary_path; + s = GetDBConfigPrimaryPath(db_primary_path); + if (!s.ok()) return s; + + std::string db_secondary_path; + s = GetDBConfigSecondaryPath(db_secondary_path); + if (!s.ok()) return s; + + std::string db_backend_url; + s = GetDBConfigBackendUrl(db_backend_url); + if (!s.ok()) return s; + + int32_t db_archive_disk_threshold; + s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold); + if (!s.ok()) return s; + + int32_t db_archive_days_threshold; + s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold); + if (!s.ok()) return s; + + int32_t db_insert_buffer_size; + s = GetDBConfigInsertBufferSize(db_insert_buffer_size); + if (!s.ok()) return s; + + int32_t db_build_index_gpu; + s = GetDBConfigBuildIndexGPU(db_build_index_gpu); + if (!s.ok()) return s; + + /* metric config */ + bool metric_enable_monitor; + s = GetMetricConfigEnableMonitor(metric_enable_monitor); + if (!s.ok()) return s; + + std::string metric_collector; + s = GetMetricConfigCollector(metric_collector); + if (!s.ok()) return s; + + std::string metric_prometheus_port; + s = GetMetricConfigPrometheusPort(metric_prometheus_port); + if (!s.ok()) return s; + + /* cache config */ + int32_t cache_cpu_mem_capacity; + s = GetCacheConfigCpuMemCapacity(cache_cpu_mem_capacity); + if (!s.ok()) return s; + + float cache_cpu_mem_threshold; + s = GetCacheConfigCpuMemThreshold(cache_cpu_mem_threshold); + if (!s.ok()) return s; + + int32_t cache_gpu_mem_capacity; + s = GetCacheConfigGpuMemCapacity(cache_gpu_mem_capacity); + if (!s.ok()) return s; + + float cache_gpu_mem_threshold; + s = GetCacheConfigGpuMemThreshold(cache_gpu_mem_threshold); + if (!s.ok()) return s; + + bool cache_insert_data; + s = GetCacheConfigCacheInsertData(cache_insert_data); + if (!s.ok()) return s; + + /* engine config */ + int32_t engine_blas_threshold; + s = GetEngineConfigBlasThreshold(engine_blas_threshold); + if (!s.ok()) return s; + + int32_t engine_omp_thread_num; + s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); + if (!s.ok()) return s; + + /* resource config */ + std::string resource_mode; + s = GetResourceConfigMode(resource_mode); + if (!s.ok()) return s; + + std::vector resource_pool; + s = GetResourceConfigPool(resource_pool); + if (!s.ok()) return s; + + return Status::OK(); +} + void Config::PrintConfigSection(const std::string& config_node_name) { std::cout << std::endl; @@ -115,8 +220,11 @@ Config::CheckServerConfigPort(const std::string &value) { Status Config::CheckServerConfigMode(const std::string &value) { - if (value != "single" && value != "cluster" && value != "read_only") { - return Status(SERVER_INVALID_ARGUMENT, "Invalid server config mode [single, cluster, read_only]: " + value); + if (value != "single" && + value != "cluster_readonly" && + value != "cluster_writable") { + return Status(SERVER_INVALID_ARGUMENT, + "Invalid server config mode [single, cluster_readonly, cluster_writable]: " + value); } return Status::OK(); } @@ -140,15 +248,15 @@ Config::CheckServerConfigTimeZone(const std::string &value) { } Status -Config::CheckDBConfigPath(const std::string &value) { +Config::CheckDBConfigPrimaryPath(const std::string &value) { if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, "DB config path empty"); + return Status(SERVER_INVALID_ARGUMENT, "DB config primary_path empty"); } return Status::OK(); } Status -Config::CheckDBConfigSlavePath(const std::string &value) { +Config::CheckDBConfigSecondaryPath(const std::string &value) { return Status::OK(); } @@ -177,15 +285,15 @@ Config::CheckDBConfigArchiveDaysThreshold(const std::string &value) { } Status -Config::CheckDBConfigBufferSize(const std::string &value) { +Config::CheckDBConfigInsertBufferSize(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config buffer_size: " + value); + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config insert_buffer_size: " + value); } else { int64_t buffer_size = std::stoi(value) * GB; unsigned long total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (buffer_size >= total_mem) { - return Status(SERVER_INVALID_ARGUMENT, "DB config buffer_size exceed system memory: " + value); + return Status(SERVER_INVALID_ARGUMENT, "DB config insert_buffer_size exceed system memory: " + value); } } return Status::OK(); @@ -205,7 +313,7 @@ Config::CheckDBConfigBuildIndexGPU(const std::string &value) { } Status -Config::CheckMetricConfigAutoBootup(const std::string& value) { +Config::CheckMetricConfigEnableMonitor(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config auto_bootup: " + value); } @@ -242,10 +350,10 @@ Config::CheckCacheConfigCpuMemCapacity(const std::string& value) { std::cerr << "Warning: cpu_mem_capacity value is too big" << std::endl; } - int32_t buffer_size; - Status s = GetDBConfigBufferSize(buffer_size); + int32_t buffer_value; + Status s = GetDBConfigInsertBufferSize(buffer_value); if (!s.ok()) return s; - int64_t insert_buffer_size = buffer_size * GB; + int64_t insert_buffer_size = buffer_value * GB; if (insert_buffer_size + cpu_cache_capacity >= total_mem) { return Status(SERVER_INVALID_ARGUMENT, "Sum of cpu_mem_capacity and buffer_size exceed system memory"); } @@ -427,23 +535,23 @@ Config::GetServerConfigStrTimeZone() { //////////////////////////////////////////////////////////////////////////////// /* db config */ std::string -Config::GetDBConfigStrPath() { +Config::GetDBConfigStrPrimaryPath() { std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PATH, - CONFIG_DB_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value).ok()) { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PRIMARY_PATH, + CONFIG_DB_PRIMARY_PATH_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); } return value; } std::string -Config::GetDBConfigStrSlavePath() { +Config::GetDBConfigStrSecondaryPath() { std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SLAVE_PATH, - CONFIG_DB_SLAVE_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value).ok()) { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SECONDARY_PATH, + CONFIG_DB_SECONDARY_PATH_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); } return value; } @@ -482,12 +590,12 @@ Config::GetDBConfigStrArchiveDaysThreshold() { } std::string -Config::GetDBConfigStrBufferSize() { +Config::GetDBConfigStrInsertBufferSize() { std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUFFER_SIZE, - CONFIG_DB_BUFFER_SIZE_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value).ok()) { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, + CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); } return value; } @@ -506,12 +614,12 @@ Config::GetDBConfigStrBuildIndexGPU() { //////////////////////////////////////////////////////////////////////////////// /* metric config */ std::string -Config::GetMetricConfigStrAutoBootup() { +Config::GetMetricConfigStrEnableMonitor() { std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { - value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_AUTO_BOOTUP, - CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value).ok()) { + value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_ENABLE_MONITOR, + CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); } return value; } @@ -659,14 +767,14 @@ Config::GetServerConfigTimeZone(std::string& value) { } Status -Config::GetDBConfigPath(std::string& value) { - value = GetDBConfigStrPath(); - return CheckDBConfigPath(value); +Config::GetDBConfigPrimaryPath(std::string& value) { + value = GetDBConfigStrPrimaryPath(); + return CheckDBConfigPrimaryPath(value); } Status -Config::GetDBConfigSlavePath(std::string& value) { - value = GetDBConfigStrSlavePath(); +Config::GetDBConfigSecondaryPath(std::string& value) { + value = GetDBConfigStrSecondaryPath(); return Status::OK(); } @@ -695,9 +803,9 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { } Status -Config::GetDBConfigBufferSize(int32_t& value) { - std::string str = GetDBConfigStrBufferSize(); - Status s = CheckDBConfigBufferSize(str); +Config::GetDBConfigInsertBufferSize(int32_t& value) { + std::string str = GetDBConfigStrInsertBufferSize(); + Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -713,9 +821,9 @@ Config::GetDBConfigBuildIndexGPU(int32_t& value) { } Status -Config::GetMetricConfigAutoBootup(bool& value) { - std::string str = GetMetricConfigStrAutoBootup(); - Status s = CheckMetricConfigPrometheusPort(str); +Config::GetMetricConfigEnableMonitor(bool& value) { + std::string str = GetMetricConfigStrEnableMonitor(); + Status s = CheckMetricConfigEnableMonitor(str); if (!s.ok()) return s; std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); @@ -847,18 +955,18 @@ Config::SetServerConfigTimeZone(const std::string& value) { /* db config */ Status -Config::SetDBConfigPath(const std::string& value) { - Status s = CheckDBConfigPath(value); +Config::SetDBConfigPrimaryPath(const std::string& value) { + Status s = CheckDBConfigPrimaryPath(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); return Status::OK(); } Status -Config::SetDBConfigSlavePath(const std::string& value) { - Status s = CheckDBConfigSlavePath(value); +Config::SetDBConfigSecondaryPath(const std::string& value) { + Status s = CheckDBConfigSecondaryPath(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); return Status::OK(); } @@ -887,10 +995,10 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { } Status -Config::SetDBConfigBufferSize(const std::string& value) { - Status s = CheckDBConfigBufferSize(value); +Config::SetDBConfigInsertBufferSize(const std::string& value) { + Status s = CheckDBConfigInsertBufferSize(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); return Status::OK(); } @@ -904,10 +1012,10 @@ Config::SetDBConfigBuildIndexGPU(const std::string& value) { /* metric config */ Status -Config::SetMetricConfigAutoBootup(const std::string& value) { - Status s = CheckMetricConfigAutoBootup(value); +Config::SetMetricConfigEnableMonitor(const std::string& value) { + Status s = CheckMetricConfigEnableMonitor(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_AUTO_BOOTUP, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value); return Status::OK(); } diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 956c08100f..710230eb0a 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -41,18 +41,18 @@ static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; /* db config */ static const char* CONFIG_DB = "db_config"; -static const char* CONFIG_DB_PATH = "path"; -static const char* CONFIG_DB_PATH_DEFAULT = "/tmp/milvus"; -static const char* CONFIG_DB_SLAVE_PATH = "slave_path"; -static const char* CONFIG_DB_SLAVE_PATH_DEFAULT = ""; +static const char* CONFIG_DB_PRIMARY_PATH = "primary_path"; +static const char* CONFIG_DB_PRIMARY_PATH_DEFAULT = "/tmp/milvus"; +static const char* CONFIG_DB_SECONDARY_PATH = "secondary_path"; +static const char* CONFIG_DB_SECONDARY_PATH_DEFAULT = ""; static const char* CONFIG_DB_BACKEND_URL = "backend_url"; static const char* CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; -static const char* CONFIG_DB_BUFFER_SIZE = "buffer_size"; -static const char* CONFIG_DB_BUFFER_SIZE_DEFAULT = "4"; +static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; +static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4"; static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; @@ -71,8 +71,8 @@ static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; /* metric config */ static const char* CONFIG_METRIC = "metric_config"; -static const char* CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup"; -static const char* CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "false"; +static const char* CONFIG_METRIC_ENABLE_MONITOR = "enable_monitor"; +static const char* CONFIG_METRIC_ENABLE_MONITOR_DEFAULT = "false"; static const char* CONFIG_METRIC_COLLECTOR = "collector"; static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config"; @@ -90,13 +90,14 @@ static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; static const char* CONFIG_RESOURCE = "resource_config"; static const char* CONFIG_RESOURCE_MODE = "mode"; static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; -static const char* CONFIG_RESOURCE_POOL = "pool"; +static const char* CONFIG_RESOURCE_POOL = "resource_pool"; class Config { public: static Config& GetInstance(); Status LoadConfigFile(const std::string& filename); + Status ValidateConfig(); void PrintAll(); private: @@ -120,16 +121,16 @@ class Config { Status CheckServerConfigTimeZone(const std::string& value); /* db config */ - Status CheckDBConfigPath(const std::string& value); - Status CheckDBConfigSlavePath(const std::string& value); + Status CheckDBConfigPrimaryPath(const std::string& value); + Status CheckDBConfigSecondaryPath(const std::string& value); Status CheckDBConfigBackendUrl(const std::string& value); Status CheckDBConfigArchiveDiskThreshold(const std::string& value); Status CheckDBConfigArchiveDaysThreshold(const std::string& value); - Status CheckDBConfigBufferSize(const std::string& value); + Status CheckDBConfigInsertBufferSize(const std::string& value); Status CheckDBConfigBuildIndexGPU(const std::string& value); /* metric config */ - Status CheckMetricConfigAutoBootup(const std::string& value); + Status CheckMetricConfigEnableMonitor(const std::string& value); Status CheckMetricConfigCollector(const std::string& value); Status CheckMetricConfigPrometheusPort(const std::string& value); @@ -156,16 +157,16 @@ class Config { std::string GetServerConfigStrTimeZone(); /* db config */ - std::string GetDBConfigStrPath(); - std::string GetDBConfigStrSlavePath(); + std::string GetDBConfigStrPrimaryPath(); + std::string GetDBConfigStrSecondaryPath(); std::string GetDBConfigStrBackendUrl(); std::string GetDBConfigStrArchiveDiskThreshold(); std::string GetDBConfigStrArchiveDaysThreshold(); - std::string GetDBConfigStrBufferSize(); + std::string GetDBConfigStrInsertBufferSize(); std::string GetDBConfigStrBuildIndexGPU(); /* metric config */ - std::string GetMetricConfigStrAutoBootup(); + std::string GetMetricConfigStrEnableMonitor(); std::string GetMetricConfigStrCollector(); std::string GetMetricConfigStrPrometheusPort(); @@ -191,16 +192,16 @@ class Config { Status GetServerConfigTimeZone(std::string& value); /* db config */ - Status GetDBConfigPath(std::string& value); - Status GetDBConfigSlavePath(std::string& value); + Status GetDBConfigPrimaryPath(std::string& value); + Status GetDBConfigSecondaryPath(std::string& value); Status GetDBConfigBackendUrl(std::string& value); Status GetDBConfigArchiveDiskThreshold(int32_t& value); Status GetDBConfigArchiveDaysThreshold(int32_t& value); - Status GetDBConfigBufferSize(int32_t& value); + Status GetDBConfigInsertBufferSize(int32_t& value); Status GetDBConfigBuildIndexGPU(int32_t& value); /* metric config */ - Status GetMetricConfigAutoBootup(bool& value); + Status GetMetricConfigEnableMonitor(bool& value); Status GetMetricConfigCollector(std::string& value); Status GetMetricConfigPrometheusPort(std::string& value); @@ -227,16 +228,16 @@ class Config { Status SetServerConfigTimeZone(const std::string& value); /* db config */ - Status SetDBConfigPath(const std::string& value); - Status SetDBConfigSlavePath(const std::string& value); + Status SetDBConfigPrimaryPath(const std::string& value); + Status SetDBConfigSecondaryPath(const std::string& value); Status SetDBConfigBackendUrl(const std::string& value); Status SetDBConfigArchiveDiskThreshold(const std::string& value); Status SetDBConfigArchiveDaysThreshold(const std::string& value); - Status SetDBConfigBufferSize(const std::string& value); + Status SetDBConfigInsertBufferSize(const std::string& value); Status SetDBConfigBuildIndexGPU(const std::string& value); /* metric config */ - Status SetMetricConfigAutoBootup(const std::string& value); + Status SetMetricConfigEnableMonitor(const std::string& value); Status SetMetricConfigCollector(const std::string& value); Status SetMetricConfigPrometheusPort(const std::string& value); diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 16b13a1249..745965dc6a 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -44,13 +44,13 @@ Status DBWrapper::StartService() { if (!s.ok()) return s; std::string path; - s = config.GetDBConfigPath(path); + s = config.GetDBConfigPrimaryPath(path); if (!s.ok()) return s; opt.meta_.path_ = path + "/db"; std::string db_slave_path; - s = config.GetDBConfigSlavePath(db_slave_path); + s = config.GetDBConfigSecondaryPath(db_slave_path); if (!s.ok()) return s; StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_); diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 57081845a5..87d3b374b2 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -241,6 +241,12 @@ Server::LoadConfig() { std::cerr << "Failed to load config file: " << config_filename_ << std::endl; exit(0); } + + s = config.ValidateConfig(); + if (!s.ok()) { + std::cerr << "Config check fail: " << s.message() << std::endl; + exit(0); + } return SERVER_SUCCESS; } From d9b6fd057540cd0abfa8a7e3a6b43ba7582f304f Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 26 Sep 2019 19:05:09 +0800 Subject: [PATCH 2/2] MS-574 rename config deploy_mode Former-commit-id: b3f2c2cc539c3cd7419c091b6eaf17df0a57f0eb --- cpp/src/db/DBImpl.cpp | 6 +++--- cpp/src/db/Options.h | 6 +++--- cpp/src/db/meta/MySQLMetaImpl.cpp | 4 ++-- cpp/src/server/Config.cpp | 26 +++++++++++++------------- cpp/src/server/Config.h | 12 ++++++------ cpp/src/server/DBWrapper.cpp | 12 ++++++------ 6 files changed, 33 insertions(+), 33 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index a763c9199c..ed1318ef1d 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -77,7 +77,7 @@ Status DBImpl::Start() { shutting_down_.store(false, std::memory_order_release); //for distribute version, some nodes are read only - if (options_.mode_ != DBOptions::MODE::READ_ONLY) { + if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { ENGINE_LOG_TRACE << "StartTimerTasks"; bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this); } @@ -98,7 +98,7 @@ Status DBImpl::Stop() { //wait compaction/buildindex finish bg_timer_thread_.join(); - if (options_.mode_ != DBOptions::MODE::READ_ONLY) { + if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { meta_ptr_->CleanUp(); } @@ -684,7 +684,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes - if (options_.mode_ == DBOptions::MODE::CLUSTER) { + if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { ttl = meta::D_SEC; } meta_ptr_->CleanUpFilesWithTTL(ttl); diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 2ba8c36a32..dbaf34b065 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -60,9 +60,9 @@ struct DBMetaOptions { struct DBOptions { typedef enum { - SINGLE, - CLUSTER, - READ_ONLY + SINGLE = 0, + CLUSTER_READONLY, + CLUSTER_WRITABLE } MODE; uint16_t merge_trigger_number_ = 2; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 3e54686ed0..5a5470008d 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -284,7 +284,7 @@ Status MySQLMetaImpl::Initialize() { //step 5: create meta tables try { - if (mode_ != DBOptions::MODE::READ_ONLY) { + if (mode_ != DBOptions::MODE::CLUSTER_READONLY) { CleanUp(); } @@ -758,7 +758,7 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { } //Scoped Connection - if (mode_ == DBOptions::MODE::CLUSTER) { + if (mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { DeleteTableFiles(table_id); } diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 3730ff6f68..31dc5fe1a9 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -84,7 +84,7 @@ Config::ValidateConfig() { if (!s.ok()) return s; std::string server_mode; - s = GetServerConfigMode(server_mode); + s = GetServerConfigDeployMode(server_mode); if (!s.ok()) return s; std::string server_time_zone; @@ -219,7 +219,7 @@ Config::CheckServerConfigPort(const std::string &value) { } Status -Config::CheckServerConfigMode(const std::string &value) { +Config::CheckServerConfigDeployMode(const std::string &value) { if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") { @@ -511,12 +511,12 @@ Config::GetServerConfigStrPort() { } std::string -Config::GetServerConfigStrMode() { +Config::GetServerConfigStrDeployMode() { std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_MODE, - CONFIG_SERVER_MODE_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value).ok()) { + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_DEPLOY_MODE, + CONFIG_SERVER_DEPLOY_MODE_DEFAULT); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); } return value; } @@ -755,9 +755,9 @@ Config::GetServerConfigPort(std::string& value) { } Status -Config::GetServerConfigMode(std::string& value) { - value = GetServerConfigStrMode(); - return CheckServerConfigMode(value); +Config::GetServerConfigDeployMode(std::string& value) { + value = GetServerConfigStrDeployMode(); + return CheckServerConfigDeployMode(value); } Status @@ -938,10 +938,10 @@ Config::SetServerConfigPort(const std::string& value) { } Status -Config::SetServerConfigMode(const std::string& value) { - Status s = CheckServerConfigMode(value); +Config::SetServerConfigDeployMode(const std::string& value) { + Status s = CheckServerConfigDeployMode(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); return Status::OK(); } diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 710230eb0a..422632a01f 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -34,8 +34,8 @@ static const char* CONFIG_SERVER_ADDRESS = "address"; static const char* CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; static const char* CONFIG_SERVER_PORT = "port"; static const char* CONFIG_SERVER_PORT_DEFAULT = "19530"; -static const char* CONFIG_SERVER_MODE = "mode"; -static const char* CONFIG_SERVER_MODE_DEFAULT = "single"; +static const char* CONFIG_SERVER_DEPLOY_MODE = "deploy_mode"; +static const char* CONFIG_SERVER_DEPLOY_MODE_DEFAULT = "single"; static const char* CONFIG_SERVER_TIME_ZONE = "time_zone"; static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; @@ -117,7 +117,7 @@ class Config { /* server config */ Status CheckServerConfigAddress(const std::string& value); Status CheckServerConfigPort(const std::string& value); - Status CheckServerConfigMode(const std::string& value); + Status CheckServerConfigDeployMode(const std::string& value); Status CheckServerConfigTimeZone(const std::string& value); /* db config */ @@ -153,7 +153,7 @@ class Config { /* server config */ std::string GetServerConfigStrAddress(); std::string GetServerConfigStrPort(); - std::string GetServerConfigStrMode(); + std::string GetServerConfigStrDeployMode(); std::string GetServerConfigStrTimeZone(); /* db config */ @@ -188,7 +188,7 @@ class Config { /* server config */ Status GetServerConfigAddress(std::string& value); Status GetServerConfigPort(std::string& value); - Status GetServerConfigMode(std::string& value); + Status GetServerConfigDeployMode(std::string& value); Status GetServerConfigTimeZone(std::string& value); /* db config */ @@ -224,7 +224,7 @@ class Config { /* server config */ Status SetServerConfigAddress(const std::string& value); Status SetServerConfigPort(const std::string& value); - Status SetServerConfigMode(const std::string& value); + Status SetServerConfigDeployMode(const std::string& value); Status SetServerConfigTimeZone(const std::string& value); /* db config */ diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 745965dc6a..49c4cac8c8 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -60,20 +60,20 @@ Status DBWrapper::StartService() { if (!s.ok()) return s; std::string mode; - s = config.GetServerConfigMode(mode); + s = config.GetServerConfigDeployMode(mode); if (!s.ok()) return s; if (mode == "single") { opt.mode_ = engine::DBOptions::MODE::SINGLE; } - else if (mode == "cluster") { - opt.mode_ = engine::DBOptions::MODE::CLUSTER; + else if (mode == "cluster_readonly") { + opt.mode_ = engine::DBOptions::MODE::CLUSTER_READONLY; } - else if (mode == "read_only") { - opt.mode_ = engine::DBOptions::MODE::READ_ONLY; + else if (mode == "cluster_writable") { + opt.mode_ = engine::DBOptions::MODE::CLUSTER_WRITABLE; } else { - std::cerr << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl; + std::cerr << "ERROR: mode specified in server_config must be ['single', 'cluster_readonly', 'cluster_writable']" << std::endl; kill(0, SIGUSR1); }