diff --git a/CHANGELOG.md b/CHANGELOG.md index 5211099105..7bf98a3635 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,7 +70,6 @@ Please mark all change in change log and use the issue from GitHub ## Task - # Milvus 0.8.0 (2020-04-15) ## Bug diff --git a/core/conf/demo/server_config.yaml b/core/conf/demo/server_config.yaml index d72eda34d0..ced4fc60b0 100644 --- a/core/conf/demo/server_config.yaml +++ b/core/conf/demo/server_config.yaml @@ -68,8 +68,8 @@ db_config: # secondary_path | A semicolon-separated list of secondary directories used | Path | | # | to save vector data and index data. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# file_cleanup_timeout | time gap between soft-delete and hard-delete | Integer | 10 (s) | -# | range [0, 3600] | | | +# file_cleanup_timeout | The time gap between marking a file as 'deleted' and | Integer | 10 (s) | +# | physically deleting this file from disk, range [0, 3600] | | | #----------------------+------------------------------------------------------------+------------+-----------------+ storage_config: primary_path: /var/lib/milvus @@ -206,14 +206,12 @@ wal_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # fatal.enable | Whether to enable fatal level logging in Milvus. | Boolean | true | #----------------------+------------------------------------------------------------+------------+-----------------+ -# path | Location of logs files. | String | | +# path | Absolute path to the folder holding the log files. | String | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# max_log_file_size | Max size of a single log file. After exceeding this value, | Integer | 256 (MB) | -# | rename this file to xxx.log.n, means the nth file. | | | +# max_log_file_size | The maximum size of each log file, size range [512, 4096] | Integer | 1024 (MB) | #----------------------+------------------------------------------------------------+------------+-----------------+ -# delete_exceeds | Milvus will keep up to ${delete_exceeds} log files per | Integer | 10 | -# | level. For example, after xxx.log.11 file is generated, | | | -# | the xxx.log.1 file will be deleted. | | | +# log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 | +# | logging level, num range [0, 1024], 0 means unlimited. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ logs: trace.enable: true @@ -223,6 +221,6 @@ logs: error.enable: true fatal.enable: true path: /var/lib/milvus/logs - max_log_file_size: 256 - delete_exceeds: 10 + max_log_file_size: 1024 + log_rotate_num: 0 diff --git a/core/conf/server_config.template b/core/conf/server_config.template index 85115390cc..215553eb30 100644 --- a/core/conf/server_config.template +++ b/core/conf/server_config.template @@ -68,8 +68,8 @@ db_config: # secondary_path | A semicolon-separated list of secondary directories used | Path | | # | to save vector data and index data. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# file_cleanup_timeout | time gap between soft-delete and hard-delete | Integer | 10 (s) | -# | range [0, 3600] | | | +# file_cleanup_timeout | The time gap between marking a file as 'deleted' and | Integer | 10 (s) | +# | physically deleting this file from disk, range [0, 3600] | | | #----------------------+------------------------------------------------------------+------------+-----------------+ storage_config: primary_path: @MILVUS_DB_PATH@ @@ -206,14 +206,12 @@ wal_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # fatal.enable | Whether to enable fatal level logging in Milvus. | Boolean | true | #----------------------+------------------------------------------------------------+------------+-----------------+ -# path | Location of logs files. | String | | +# path | Absolute path to the folder holding the log files. | String | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# max_log_file_size | Max size of a single log file. After exceeding this value, | Integer | 256 (MB) | -# | rename this file to xxx.log.n, means the nth file. | | | +# max_log_file_size | The maximum size of each log file, size range [512, 4096] | Integer | 1024 (MB) | #----------------------+------------------------------------------------------------+------------+-----------------+ -# delete_exceeds | Milvus will keep up to ${delete_exceeds} log files per | Integer | 10 | -# | level. For example, after xxx.log.11 file is generated, | | | -# | the xxx.log.1 file will be deleted. | | | +# log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 | +# | logging level, num range [0, 1024], 0 means unlimited. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ logs: trace.enable: true @@ -223,6 +221,6 @@ logs: error.enable: true fatal.enable: true path: @MILVUS_DB_PATH@/logs - max_log_file_size: 256 - delete_exceeds: 10 + max_log_file_size: 1024 + log_rotate_num: 0 diff --git a/core/src/config/Config.cpp b/core/src/config/Config.cpp index e1e757fcf1..9f78b1f3d4 100644 --- a/core/src/config/Config.cpp +++ b/core/src/config/Config.cpp @@ -76,6 +76,8 @@ const char* CONFIG_STORAGE_SECONDARY_PATH = "secondary_path"; const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT = ""; const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT = "file_cleanup_timeout"; const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_DEFAULT = "10"; +const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN = 0; +const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MAX = 3600; // const char* CONFIG_STORAGE_S3_ENABLE = "s3_enable"; // const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT = "false"; // const char* CONFIG_STORAGE_S3_ADDRESS = "s3_address"; @@ -150,8 +152,8 @@ const char* CONFIG_WAL_RECOVERY_ERROR_IGNORE = "recovery_error_ignore"; const char* CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT = "true"; const char* CONFIG_WAL_BUFFER_SIZE = "buffer_size"; const char* CONFIG_WAL_BUFFER_SIZE_DEFAULT = "256"; -const int64_t CONFIG_WAL_BUFFER_SIZE_MAX = 4096; const int64_t CONFIG_WAL_BUFFER_SIZE_MIN = 64; +const int64_t CONFIG_WAL_BUFFER_SIZE_MAX = 4096; const char* CONFIG_WAL_WAL_PATH = "wal_path"; const char* CONFIG_WAL_WAL_PATH_DEFAULT = "/tmp/milvus/wal"; @@ -172,13 +174,13 @@ const char* CONFIG_LOGS_FATAL_ENABLE_DEFAULT = "true"; const char* CONFIG_LOGS_PATH = "path"; const char* CONFIG_LOGS_PATH_DEFAULT = "/tmp/milvus/logs"; const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE = "max_log_file_size"; -const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT = "256"; -const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX = 512; -const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN = 64; -const char* CONFIG_LOGS_DELETE_EXCEEDS = "delete_exceeds"; -const char* CONFIG_LOGS_DELETE_EXCEEDS_DEFAULT = "10"; -const int64_t CONFIG_LOGS_DELETE_EXCEEDS_MAX = 4096; -const int64_t CONFIG_LOGS_DELETE_EXCEEDS_MIN = 1; +const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT = "1024"; +const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN = 512; +const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX = 4096; +const char* CONFIG_LOGS_LOG_ROTATE_NUM = "log_rotate_num"; +const char* CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT = "0"; +const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MIN = 0; +const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MAX = 1024; constexpr int64_t GB = 1UL << 30; constexpr int32_t PORT_NUMBER_MIN = 1024; @@ -416,8 +418,8 @@ Config::ValidateConfig() { int64_t logs_max_log_file_size; STATUS_CHECK(GetLogsMaxLogFileSize(logs_max_log_file_size)); - int64_t delete_exceeds; - STATUS_CHECK(GetLogsDeleteExceeds(delete_exceeds)); + int64_t logs_log_rotate_num; + STATUS_CHECK(GetLogsLogRotateNum(logs_log_rotate_num)); return Status::OK(); } @@ -493,7 +495,7 @@ Config::ResetDefaultConfig() { STATUS_CHECK(SetLogsFatalEnable(CONFIG_LOGS_FATAL_ENABLE_DEFAULT)); STATUS_CHECK(SetLogsPath(CONFIG_LOGS_PATH_DEFAULT)); STATUS_CHECK(SetLogsMaxLogFileSize(CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT)); - STATUS_CHECK(SetLogsDeleteExceeds(CONFIG_LOGS_DELETE_EXCEEDS_DEFAULT)); + STATUS_CHECK(SetLogsLogRotateNum(CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT)); return Status::OK(); } @@ -635,6 +637,28 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key } else { status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } + } else if (parent_key == CONFIG_LOGS) { + if (child_key == CONFIG_LOGS_TRACE_ENABLE) { + status = SetLogsTraceEnable(value); + } else if (child_key == CONFIG_LOGS_DEBUG_ENABLE) { + status = SetLogsDebugEnable(value); + } else if (child_key == CONFIG_LOGS_INFO_ENABLE) { + status = SetLogsInfoEnable(value); + } else if (child_key == CONFIG_LOGS_WARNING_ENABLE) { + status = SetLogsWarningEnable(value); + } else if (child_key == CONFIG_LOGS_ERROR_ENABLE) { + status = SetLogsErrorEnable(value); + } else if (child_key == CONFIG_LOGS_FATAL_ENABLE) { + status = SetLogsFatalEnable(value); + } else if (child_key == CONFIG_LOGS_PATH) { + status = SetLogsPath(value); + } else if (child_key == CONFIG_LOGS_MAX_LOG_FILE_SIZE) { + status = SetLogsMaxLogFileSize(value); + } else if (child_key == CONFIG_LOGS_LOG_ROTATE_NUM) { + status = SetLogsLogRotateNum(value); + } else { + status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); + } } if (status.ok()) { @@ -1086,23 +1110,18 @@ Config::CheckStorageConfigSecondaryPath(const std::string& value) { Status Config::CheckStorageConfigFileCleanupTimeout(const std::string& value) { - auto status = Status::OK(); - - if (value.empty()) { - return status; - } - if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid file cleanup timeout: " + value + + std::string msg = "Invalid file_cleanup_timeout: " + value + ". Possible reason: storage_config.file_cleanup_timeout is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - const int64_t min = 0, max = 3600; int64_t file_cleanup_timeout = std::stoll(value); - if (file_cleanup_timeout < min || file_cleanup_timeout > max) { - std::string msg = "Invalid file cleanup timeout: " + value + + if (file_cleanup_timeout < CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN || + file_cleanup_timeout > CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MAX) { + std::string msg = "Invalid file_cleanup_timeout: " + value + ". Possible reason: storage_config.file_cleanup_timeout is not in range [" + - std::to_string(min) + ", " + std::to_string(max) + "]."; + std::to_string(CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN) + ", " + + std::to_string(CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN) + "]."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -1673,19 +1692,38 @@ Config::CheckLogsMaxLogFileSize(const std::string& value) { std::string msg = "Invalid max_log_file_size: " + value + ". Possible reason: logs.max_log_file_size is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + int64_t max_log_file_size = std::stoll(value); + if (max_log_file_size < CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN || + max_log_file_size > CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX) { + std::string msg = "Invalid max_log_file_size: " + value + + ". Possible reason: logs.max_log_file_size is not in range [" + + std::to_string(CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN) + ", " + + std::to_string(CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX) + "]."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } } return Status::OK(); } Status -Config::CheckLogsDeleteExceeds(const std::string& value) { +Config::CheckLogsLogRotateNum(const std::string& value) { auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok(); - fiu_do_on("check_logs_delete_exceeds_fail", exist_error = true); + fiu_do_on("check_logs_log_rotate_num_fail", exist_error = true); if (exist_error) { - std::string msg = "Invalid max_log_file_size: " + value + - ". Possible reason: logs.max_log_file_size is not a positive integer."; + std::string msg = + "Invalid log_rotate_num: " + value + ". Possible reason: logs.log_rotate_num is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + int64_t log_rotate_num = std::stoll(value); + if (log_rotate_num < CONFIG_LOGS_LOG_ROTATE_NUM_MIN || log_rotate_num > CONFIG_LOGS_LOG_ROTATE_NUM_MAX) { + std::string msg = "Invalid log_rotate_num: " + value + + ". Possible reason: logs.log_rotate_num is not in range [" + + std::to_string(CONFIG_LOGS_LOG_ROTATE_NUM_MIN) + ", " + + std::to_string(CONFIG_LOGS_LOG_ROTATE_NUM_MAX) + "]."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } } return Status::OK(); } @@ -2221,30 +2259,14 @@ Config::GetLogsMaxLogFileSize(int64_t& value) { std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_MAX_LOG_FILE_SIZE, CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT); STATUS_CHECK(CheckLogsMaxLogFileSize(str)); value = std::stoll(str); - if (value == 0) { - // OFF - } else if (value > CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX) { - value = CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX; - } else if (value < CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN) { - value = CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN; - } - return Status::OK(); } Status -Config::GetLogsDeleteExceeds(int64_t& value) { - std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_DELETE_EXCEEDS, CONFIG_LOGS_DELETE_EXCEEDS_DEFAULT); - STATUS_CHECK(CheckLogsDeleteExceeds(str)); +Config::GetLogsLogRotateNum(int64_t& value) { + std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_LOG_ROTATE_NUM, CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT); + STATUS_CHECK(CheckLogsLogRotateNum(str)); value = std::stoll(str); - if (value == 0) { - // OFF - } else if (value > CONFIG_LOGS_DELETE_EXCEEDS_MAX) { - value = CONFIG_LOGS_DELETE_EXCEEDS_MAX; - } else if (value < CONFIG_LOGS_DELETE_EXCEEDS_MIN) { - value = CONFIG_LOGS_DELETE_EXCEEDS_MIN; - } - return Status::OK(); } @@ -2578,9 +2600,9 @@ Config::SetLogsMaxLogFileSize(const std::string& value) { } Status -Config::SetLogsDeleteExceeds(const std::string& value) { - STATUS_CHECK(CheckLogsDeleteExceeds(value)); - return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_DELETE_EXCEEDS, value); +Config::SetLogsLogRotateNum(const std::string& value) { + STATUS_CHECK(CheckLogsLogRotateNum(value)); + return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_LOG_ROTATE_NUM, value); } } // namespace server diff --git a/core/src/config/Config.h b/core/src/config/Config.h index 921b7193ef..714a2216d0 100644 --- a/core/src/config/Config.h +++ b/core/src/config/Config.h @@ -63,6 +63,8 @@ extern const char* CONFIG_STORAGE_PRIMARY_PATH_DEFAULT; extern const char* CONFIG_STORAGE_SECONDARY_PATH; extern const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT; extern const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT; +extern const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN; +extern const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MAX; // extern const char* CONFIG_STORAGE_S3_ENABLE; // extern const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT; // extern const char* CONFIG_STORAGE_S3_ADDRESS; @@ -133,8 +135,8 @@ extern const char* CONFIG_WAL_RECOVERY_ERROR_IGNORE; extern const char* CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT; extern const char* CONFIG_WAL_BUFFER_SIZE; extern const char* CONFIG_WAL_BUFFER_SIZE_DEFAULT; -extern const int64_t CONFIG_WAL_BUFFER_SIZE_MAX; extern const int64_t CONFIG_WAL_BUFFER_SIZE_MIN; +extern const int64_t CONFIG_WAL_BUFFER_SIZE_MAX; extern const char* CONFIG_WAL_WAL_PATH; extern const char* CONFIG_WAL_WAL_PATH_DEFAULT; @@ -155,12 +157,12 @@ extern const char* CONFIG_LOGS_FATAL_ENABLE_DEFAULT; extern const char* CONFIG_LOGS_PATH; extern const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE; extern const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT; -extern const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX; extern const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN; -extern const char* CONFIG_LOGS_DELETE_EXCEEDS; -extern const char* CONFIG_LOGS_DELETE_EXCEEDS_DEFAULT; -extern const int64_t CONFIG_LOGS_DELETE_EXCEEDS_MAX; -extern const int64_t CONFIG_LOGS_DELETE_EXCEEDS_MIN; +extern const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX; +extern const char* CONFIG_LOGS_LOG_ROTATE_NUM; +extern const char* CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT; +extern const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MIN; +extern const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MAX; class Config { private: @@ -334,7 +336,7 @@ class Config { Status CheckLogsMaxLogFileSize(const std::string& value); Status - CheckLogsDeleteExceeds(const std::string& value); + CheckLogsLogRotateNum(const std::string& value); std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); @@ -469,7 +471,7 @@ class Config { Status GetLogsMaxLogFileSize(int64_t& value); Status - GetLogsDeleteExceeds(int64_t& value); + GetLogsLogRotateNum(int64_t& value); Status GetServerRestartRequired(bool& required); @@ -595,7 +597,7 @@ class Config { Status SetLogsMaxLogFileSize(const std::string& value); Status - SetLogsDeleteExceeds(const std::string& value); + SetLogsLogRotateNum(const std::string& value); private: bool restart_required_ = false; diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index 31f6f14a89..b528d517cb 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -217,7 +217,7 @@ Server::Start() { STATUS_CHECK(config.GetLogsFatalEnable(fatal_enable)); STATUS_CHECK(config.GetLogsPath(logs_path)); STATUS_CHECK(config.GetLogsMaxLogFileSize(max_log_file_size)); - STATUS_CHECK(config.GetLogsDeleteExceeds(delete_exceeds)); + STATUS_CHECK(config.GetLogsLogRotateNum(delete_exceeds)); InitLog(trace_enable, debug_enable, info_enable, warning_enable, error_enable, fatal_enable, logs_path, max_log_file_size, delete_exceeds); } diff --git a/core/src/utils/LogUtil.cpp b/core/src/utils/LogUtil.cpp index 617e8dbb03..8c9f486a3e 100644 --- a/core/src/utils/LogUtil.cpp +++ b/core/src/utils/LogUtil.cpp @@ -32,6 +32,7 @@ static int trace_idx = 0; static int error_idx = 0; static int fatal_idx = 0; static int64_t logs_delete_exceeds = 1; +static bool enable_log_delete = false; } // namespace // TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename @@ -61,7 +62,7 @@ RolloutHandler(const char* filename, std::size_t size, el::Level level) { case el::Level::Debug: { s.append("." + std::to_string(++debug_idx)); ret = rename(m.c_str(), s.c_str()); - if (debug_idx - logs_delete_exceeds > 0) { + if (enable_log_delete && debug_idx - logs_delete_exceeds > 0) { std::string to_delete = m + "." + std::to_string(debug_idx - logs_delete_exceeds); // std::cout << "remote " << to_delete << std::endl; boost::filesystem::remove(to_delete); @@ -71,7 +72,7 @@ RolloutHandler(const char* filename, std::size_t size, el::Level level) { case el::Level::Warning: { s.append("." + std::to_string(++warning_idx)); ret = rename(m.c_str(), s.c_str()); - if (warning_idx - logs_delete_exceeds > 0) { + if (enable_log_delete && warning_idx - logs_delete_exceeds > 0) { std::string to_delete = m + "." + std::to_string(warning_idx - logs_delete_exceeds); boost::filesystem::remove(to_delete); } @@ -80,7 +81,7 @@ RolloutHandler(const char* filename, std::size_t size, el::Level level) { case el::Level::Trace: { s.append("." + std::to_string(++trace_idx)); ret = rename(m.c_str(), s.c_str()); - if (trace_idx - logs_delete_exceeds > 0) { + if (enable_log_delete && trace_idx - logs_delete_exceeds > 0) { std::string to_delete = m + "." + std::to_string(trace_idx - logs_delete_exceeds); boost::filesystem::remove(to_delete); } @@ -89,7 +90,7 @@ RolloutHandler(const char* filename, std::size_t size, el::Level level) { case el::Level::Error: { s.append("." + std::to_string(++error_idx)); ret = rename(m.c_str(), s.c_str()); - if (error_idx - logs_delete_exceeds > 0) { + if (enable_log_delete && error_idx - logs_delete_exceeds > 0) { std::string to_delete = m + "." + std::to_string(error_idx - logs_delete_exceeds); boost::filesystem::remove(to_delete); } @@ -98,7 +99,7 @@ RolloutHandler(const char* filename, std::size_t size, el::Level level) { case el::Level::Fatal: { s.append("." + std::to_string(++fatal_idx)); ret = rename(m.c_str(), s.c_str()); - if (fatal_idx - logs_delete_exceeds > 0) { + if (enable_log_delete && fatal_idx - logs_delete_exceeds > 0) { std::string to_delete = m + "." + std::to_string(fatal_idx - logs_delete_exceeds); boost::filesystem::remove(to_delete); } @@ -107,7 +108,7 @@ RolloutHandler(const char* filename, std::size_t size, el::Level level) { default: { s.append("." + std::to_string(++global_idx)); ret = rename(m.c_str(), s.c_str()); - if (global_idx - logs_delete_exceeds > 0) { + if (enable_log_delete && global_idx - logs_delete_exceeds > 0) { std::string to_delete = m + "." + std::to_string(global_idx - logs_delete_exceeds); boost::filesystem::remove(to_delete); } @@ -183,26 +184,29 @@ InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_ena defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "false"); } - // set max_log_file_size = 0 means disable log file rotating - if (max_log_file_size != 0) { - if (max_log_file_size < 64 || max_log_file_size > 512) { - return Status(SERVER_UNEXPECTED_ERROR, - "max_log_file_size must in range[64, 512], now is " + std::to_string(max_log_file_size)); - } - max_log_file_size *= 1024 * 1024; - defaultConf.setGlobally(el::ConfigurationType::MaxLogFileSize, std::to_string(max_log_file_size)); - el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck); - el::Helpers::installPreRollOutCallback(RolloutHandler); - el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog); + if (max_log_file_size < CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN || + max_log_file_size > CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX) { + return Status(SERVER_UNEXPECTED_ERROR, "max_log_file_size must in range[" + + std::to_string(CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN) + ", " + + std::to_string(CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX) + "], now is " + + std::to_string(max_log_file_size)); + } + max_log_file_size *= 1024 * 1024; + defaultConf.setGlobally(el::ConfigurationType::MaxLogFileSize, std::to_string(max_log_file_size)); + el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck); + el::Helpers::installPreRollOutCallback(RolloutHandler); + el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog); - // set delete_exceeds = 0 means disable throw away log file even they reach certain limit. - if (delete_exceeds != 0) { - if (delete_exceeds < 1 || delete_exceeds > 4096) { - return Status(SERVER_UNEXPECTED_ERROR, - "delete_exceeds must in range[1, 4096], now is " + std::to_string(delete_exceeds)); - } - logs_delete_exceeds = delete_exceeds; + // set delete_exceeds = 0 means disable throw away log file even they reach certain limit. + if (delete_exceeds != 0) { + if (delete_exceeds < CONFIG_LOGS_LOG_ROTATE_NUM_MIN || delete_exceeds > CONFIG_LOGS_LOG_ROTATE_NUM_MAX) { + return Status(SERVER_UNEXPECTED_ERROR, "delete_exceeds must in range[" + + std::to_string(CONFIG_LOGS_LOG_ROTATE_NUM_MIN) + ", " + + std::to_string(CONFIG_LOGS_LOG_ROTATE_NUM_MAX) + "], now is " + + std::to_string(delete_exceeds)); } + enable_log_delete = true; + logs_delete_exceeds = delete_exceeds; } el::Loggers::reconfigureLogger("default", defaultConf); diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 98fbd24abc..99cc110b4d 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -134,7 +134,6 @@ TEST_F(ConfigTest, CONFIG_TEST) { } TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { - std::string config_path(CONFIG_PATH); milvus::server::Config& config = milvus::server::Config::GetInstance(); std::string str_val; int64_t int64_val; @@ -359,6 +358,52 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(config.SetWalConfigWalPath(wal_path).ok()); ASSERT_TRUE(config.GetWalConfigWalPath(str_val).ok()); ASSERT_TRUE(str_val == wal_path); + + /* logs config */ + bool logs_trace_enable = false; + ASSERT_TRUE(config.SetLogsTraceEnable(std::to_string(logs_trace_enable)).ok()); + ASSERT_TRUE(config.GetLogsTraceEnable(bool_val).ok()); + ASSERT_TRUE(bool_val == logs_trace_enable); + + bool logs_debug_enable = false; + ASSERT_TRUE(config.SetLogsDebugEnable(std::to_string(logs_debug_enable)).ok()); + ASSERT_TRUE(config.GetLogsDebugEnable(bool_val).ok()); + ASSERT_TRUE(bool_val == logs_debug_enable); + + bool logs_info_enable = false; + ASSERT_TRUE(config.SetLogsTraceEnable(std::to_string(logs_info_enable)).ok()); + ASSERT_TRUE(config.GetLogsTraceEnable(bool_val).ok()); + ASSERT_TRUE(bool_val == logs_info_enable); + + bool logs_warning_enable = false; + ASSERT_TRUE(config.SetLogsDebugEnable(std::to_string(logs_warning_enable)).ok()); + ASSERT_TRUE(config.GetLogsDebugEnable(bool_val).ok()); + ASSERT_TRUE(bool_val == logs_warning_enable); + + bool logs_error_enable = false; + ASSERT_TRUE(config.SetLogsTraceEnable(std::to_string(logs_error_enable)).ok()); + ASSERT_TRUE(config.GetLogsTraceEnable(bool_val).ok()); + ASSERT_TRUE(bool_val == logs_error_enable); + + bool logs_fatal_enable = false; + ASSERT_TRUE(config.SetLogsDebugEnable(std::to_string(logs_fatal_enable)).ok()); + ASSERT_TRUE(config.GetLogsDebugEnable(bool_val).ok()); + ASSERT_TRUE(bool_val == logs_fatal_enable); + + std::string logs_path = "/tmp/aaa/logs"; + ASSERT_TRUE(config.SetLogsPath(logs_path).ok()); + ASSERT_TRUE(config.GetLogsPath(str_val).ok()); + ASSERT_TRUE(str_val == logs_path); + + int64_t logs_max_log_file_size = 1000; + ASSERT_TRUE(config.SetLogsMaxLogFileSize(std::to_string(logs_max_log_file_size)).ok()); + ASSERT_TRUE(config.GetLogsMaxLogFileSize(int64_val).ok()); + ASSERT_TRUE(int64_val == logs_max_log_file_size); + + int64_t logs_log_rotate_num = 100; + ASSERT_TRUE(config.SetLogsLogRotateNum(std::to_string(logs_log_rotate_num)).ok()); + ASSERT_TRUE(config.GetLogsLogRotateNum(int64_val).ok()); + ASSERT_TRUE(int64_val == logs_log_rotate_num); } std::string @@ -374,181 +419,198 @@ gen_set_command(const std::string& parent_node, const std::string& child_node, c } TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { -// std::string config_path(CONFIG_PATH); -// milvus::Status s; -// -// std::string conf_file = std::string(CONFIG_PATH) + VALID_CONFIG_FILE; -// milvus::server::Config& config = milvus::server::Config::GetInstance(); -// -// auto status = config.LoadConfigFile(conf_file); -// ASSERT_TRUE(status.ok()) << status.message(); -// -// std::string get_cmd, set_cmd; -// std::string result, dummy; -// -// s = config.ProcessConfigCli(result, "get_config *"); -// ASSERT_TRUE(s.ok()); -// -// /* server config */ -// std::string server_addr = "192.168.1.155"; -// get_cmd = gen_get_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS); -// set_cmd = gen_set_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS, server_addr); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// -// /* db config */ -// std::string db_backend_url = "sqlite://milvus:zilliz@:/"; -// get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL); -// set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL, db_backend_url); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// -// /* metric config */ -// std::string metric_enable_monitor = "false"; -// get_cmd = gen_get_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR); -// set_cmd = gen_set_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR, metric_enable_monitor); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// -// /* storage config */ -// std::string storage_primary_path = "/tmp/milvus1"; -// get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH); -// set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH, storage_primary_path); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// -// /* cache config */ -// std::string cache_cpu_cache_capacity = "1"; -// get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY); -// set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY, cache_cpu_cache_capacity); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// ASSERT_TRUE(result == cache_cpu_cache_capacity); -// -// std::string cache_cpu_cache_threshold = "0.1"; -// get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD); -// set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD, cache_cpu_cache_threshold); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// ASSERT_TRUE(result == cache_cpu_cache_threshold); -// -// std::string cache_insert_buffer_size = "1"; -// get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_INSERT_BUFFER_SIZE); -// set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_INSERT_BUFFER_SIZE, cache_insert_buffer_size); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// ASSERT_TRUE(result == cache_insert_buffer_size); -// -// std::string cache_insert_data = "true"; -// get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA); -// set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA, cache_insert_data); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// ASSERT_TRUE(result == cache_insert_data); -// -// /* engine config */ -// std::string engine_use_blas_threshold = "50"; -// get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD); -// set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD, engine_use_blas_threshold); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// ASSERT_TRUE(result == engine_use_blas_threshold); -// -// std::string engine_omp_thread_num = "1"; -// get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM); -// set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM, engine_omp_thread_num); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// ASSERT_TRUE(result == engine_omp_thread_num); -// -// std::string engine_simd_type = "sse"; -// get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_SIMD_TYPE); -// set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_SIMD_TYPE, engine_simd_type); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// ASSERT_TRUE(result == engine_simd_type); -// -//#ifdef MILVUS_GPU_VERSION -// std::string engine_gpu_search_threshold = "800"; -// get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD); -// set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, engine_gpu_search_threshold); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// ASSERT_TRUE(result == engine_gpu_search_threshold); -//#endif -// -// /* gpu resource config */ -//#ifdef MILVUS_GPU_VERSION -// std::string resource_enable_gpu = "true"; -// get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE); -// set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE, resource_enable_gpu); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// ASSERT_TRUE(result == resource_enable_gpu); -// -// std::string gpu_cache_capacity = "1"; -// get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY); -// set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY, gpu_cache_capacity); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// ASSERT_TRUE(result == gpu_cache_capacity); -// -// std::string gpu_cache_threshold = "0.2"; -// get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD); -// set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, gpu_cache_threshold); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// ASSERT_TRUE(result == gpu_cache_threshold); -// -// std::string search_resources = "gpu0"; -// get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES); -// set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, search_resources); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// ASSERT_TRUE(result == search_resources); -// -// std::string build_index_resources = "gpu0"; -// get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES); -// set_cmd = -// gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, build_index_resources); -// s = config.ProcessConfigCli(dummy, set_cmd); -// ASSERT_TRUE(s.ok()); -// s = config.ProcessConfigCli(result, get_cmd); -// ASSERT_TRUE(s.ok()); -// ASSERT_TRUE(result == build_index_resources); -//#endif + std::string conf_file = std::string(CONFIG_PATH) + VALID_CONFIG_FILE; + milvus::server::Config& config = milvus::server::Config::GetInstance(); + + auto s = config.LoadConfigFile(conf_file); + ASSERT_TRUE(s.ok()) << s.message(); + s = config.ResetDefaultConfig(); + ASSERT_TRUE(s.ok()); + + std::string get_cmd, set_cmd; + std::string result, dummy; + + s = config.ProcessConfigCli(result, "get_config *"); + ASSERT_TRUE(s.ok()); + + /* server config */ + std::string server_addr = "192.168.1.155"; + get_cmd = gen_get_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS); + set_cmd = gen_set_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS, server_addr); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + + /* db config */ + std::string db_backend_url = "sqlite://milvus:zilliz@:/"; + get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL); + set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL, db_backend_url); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + + /* metric config */ + std::string metric_enable_monitor = "false"; + get_cmd = gen_get_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR); + set_cmd = gen_set_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR, metric_enable_monitor); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + + /* storage config */ + std::string storage_primary_path = "/tmp/milvus1"; + get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH); + set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH, storage_primary_path); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + + /* cache config */ + std::string cache_cpu_cache_capacity = "1"; + get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY); + set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY, cache_cpu_cache_capacity); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == cache_cpu_cache_capacity); + + std::string cache_cpu_cache_threshold = "0.1"; + get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD); + set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD, cache_cpu_cache_threshold); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == cache_cpu_cache_threshold); + + std::string cache_insert_buffer_size = "1"; + get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_INSERT_BUFFER_SIZE); + set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_INSERT_BUFFER_SIZE, cache_insert_buffer_size); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == cache_insert_buffer_size); + + std::string cache_insert_data = "true"; + get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA); + set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA, cache_insert_data); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == cache_insert_data); + + /* engine config */ + std::string engine_use_blas_threshold = "50"; + get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD); + set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD, engine_use_blas_threshold); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == engine_use_blas_threshold); + + std::string engine_omp_thread_num = "1"; + get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM); + set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM, engine_omp_thread_num); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == engine_omp_thread_num); + + std::string engine_simd_type = "sse"; + get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_SIMD_TYPE); + set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_SIMD_TYPE, engine_simd_type); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == engine_simd_type); + +#ifdef MILVUS_GPU_VERSION + std::string engine_gpu_search_threshold = "800"; + get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD); + set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, engine_gpu_search_threshold); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == engine_gpu_search_threshold); +#endif + + /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION + std::string resource_enable_gpu = "true"; + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE); + set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE, resource_enable_gpu); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == resource_enable_gpu); + + std::string gpu_cache_capacity = "1"; + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY); + set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY, gpu_cache_capacity); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == gpu_cache_capacity); + + std::string gpu_cache_threshold = "0.2"; + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD); + set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, gpu_cache_threshold); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == gpu_cache_threshold); + + std::string search_resources = "gpu0"; + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES); + set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, search_resources); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == search_resources); + + std::string build_index_resources = "gpu0"; + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES); + set_cmd = + gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, build_index_resources); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == build_index_resources); +#endif + + /* wal config */ + std::string wal_path = "/tmp/aaa/wal"; + get_cmd = gen_get_command(ms::CONFIG_WAL, ms::CONFIG_WAL_WAL_PATH); + set_cmd = gen_set_command(ms::CONFIG_WAL, ms::CONFIG_WAL_WAL_PATH, wal_path); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + + /* logs config */ + std::string logs_path = "/tmp/aaa/logs"; + get_cmd = gen_get_command(ms::CONFIG_LOGS, ms::CONFIG_LOGS_PATH); + set_cmd = gen_set_command(ms::CONFIG_LOGS, ms::CONFIG_LOGS_PATH, logs_path); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); } TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { @@ -674,6 +736,19 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(config.SetWalConfigWalPath("").ok()); ASSERT_FALSE(config.SetWalConfigBufferSize("-1").ok()); ASSERT_FALSE(config.SetWalConfigBufferSize("a").ok()); + + /* wal config */ + ASSERT_FALSE(config.SetLogsTraceEnable("invalid").ok()); + ASSERT_FALSE(config.SetLogsDebugEnable("invalid").ok()); + ASSERT_FALSE(config.SetLogsInfoEnable("invalid").ok()); + ASSERT_FALSE(config.SetLogsWarningEnable("invalid").ok()); + ASSERT_FALSE(config.SetLogsErrorEnable("invalid").ok()); + ASSERT_FALSE(config.SetLogsFatalEnable("invalid").ok()); + ASSERT_FALSE(config.SetLogsPath("").ok()); + ASSERT_FALSE(config.SetLogsMaxLogFileSize("-1").ok()); + ASSERT_FALSE(config.SetLogsMaxLogFileSize("511").ok()); + ASSERT_FALSE(config.SetLogsLogRotateNum("-1").ok()); + ASSERT_FALSE(config.SetLogsLogRotateNum("1025").ok()); } TEST_F(ConfigTest, SERVER_CONFIG_TEST) { @@ -919,6 +994,52 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) { s = config.ValidateConfig(); ASSERT_FALSE(s.ok()); fiu_disable("check_wal_path_fail"); + + /* logs config */ + fiu_enable("check_logs_trace_enable_fail", 1, NULL, 0); + s = config.ValidateConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_trace_enable_fail"); + + fiu_enable("check_logs_debug_enable_fail", 1, NULL, 0); + s = config.ValidateConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_debug_enable_fail"); + + fiu_enable("check_logs_info_enable_fail", 1, NULL, 0); + s = config.ValidateConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_info_enable_fail"); + + fiu_enable("check_logs_warning_enable_fail", 1, NULL, 0); + s = config.ValidateConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_warning_enable_fail"); + + fiu_enable("check_logs_error_enable_fail", 1, NULL, 0); + s = config.ValidateConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_error_enable_fail"); + + fiu_enable("check_logs_fatal_enable_fail", 1, NULL, 0); + s = config.ValidateConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_fatal_enable_fail"); + + fiu_enable("check_logs_path_fail", 1, NULL, 0); + s = config.ValidateConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_path_fail"); + + fiu_enable("check_logs_max_log_file_size_fail", 1, NULL, 0); + s = config.ValidateConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_max_log_file_size_fail"); + + fiu_enable("check_logs_log_rotate_num_fail", 1, NULL, 0); + s = config.ValidateConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_log_rotate_num_fail"); } TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) { @@ -1088,6 +1209,52 @@ TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) { s = config.ResetDefaultConfig(); ASSERT_FALSE(s.ok()); fiu_disable("check_wal_path_fail"); + + /* logs config */ + fiu_enable("check_logs_trace_enable_fail", 1, NULL, 0); + s = config.ResetDefaultConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_trace_enable_fail"); + + fiu_enable("check_logs_debug_enable_fail", 1, NULL, 0); + s = config.ResetDefaultConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_debug_enable_fail"); + + fiu_enable("check_logs_info_enable_fail", 1, NULL, 0); + s = config.ResetDefaultConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_info_enable_fail"); + + fiu_enable("check_logs_warning_enable_fail", 1, NULL, 0); + s = config.ResetDefaultConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_warning_enable_fail"); + + fiu_enable("check_logs_error_enable_fail", 1, NULL, 0); + s = config.ResetDefaultConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_error_enable_fail"); + + fiu_enable("check_logs_fatal_enable_fail", 1, NULL, 0); + s = config.ResetDefaultConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_fatal_enable_fail"); + + fiu_enable("check_logs_path_fail", 1, NULL, 0); + s = config.ResetDefaultConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_path_fail"); + + fiu_enable("check_logs_max_log_file_size_fail", 1, NULL, 0); + s = config.ResetDefaultConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_max_log_file_size_fail"); + + fiu_enable("check_logs_log_rotate_num_fail", 1, NULL, 0); + s = config.ResetDefaultConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_logs_log_rotate_num_fail"); } TEST_F(ConfigTest, SERVER_CONFIG_OTHER_CONFIGS_FAIL_TEST) { diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index f6b988e10d..485de6b001 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -246,7 +246,7 @@ TEST(UtilTest, BLOCKINGQUEUE_TEST) { } TEST(UtilTest, LOG_TEST) { - auto status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util", 256, 10); + auto status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util", 1024, 10); ASSERT_TRUE(status.ok()); EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer));