admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support logging to stdout (#4738)

Browse Source 
Signed-off-by: shengjun.li <shengjun.li@zilliz.com>
This commit is contained in:
shengjun.li 2021-02-24 18:54:19 +08:00 committed by GitHub
parent 3570650083
commit 10c8cd4e9f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 161 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
CHANGELOG.md
View File

@ -2,11 +2,21 @@
Please mark all change in change log and use the issue from GitHub
# Milvus 0.10.6 (TBD)
# Milvus 1.0.0-beta (TBD)
## Bug
## Feature
- \#3977 Support logging to stdout
## Improvement
## Task
# Milvus 0.10.6 (2021-02-23)
## Bug
- \#4683 A negative zero may be returned if the metric_type is Tanimoto
- \#4678 Server crash on BinaryFlat if dimension is not a power of 2
- \#4720 The GPU cache holds much more data than the configured value
- \#4719 The GPU cache holds much more data than the configured value
## Feature

6
core/conf/demo/server_config.yaml
View File

@ -179,12 +179,18 @@ fpga:
# log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 |
# | logging level, num range [0, 1024], 0 means unlimited. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# log_to_stdout | Whether to write logs to standard output in Milvus. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# log_to_file | Whether to write logs to files in Milvus | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
logs:
level: debug
trace.enable: true
path: /var/lib/milvus/logs
max_log_file_size: 1024MB
log_rotate_num: 0
log_to_stdout: false
log_to_file: true
#----------------------+------------------------------------------------------------+------------+-----------------+
# Metric Config | Description | Type | Default |

8
core/conf/server_config.template
View File

@ -159,11 +159,11 @@ gpu:
# search_devices | The list of FPGA devices used for search computation. | DeviceList | fpga0 |
# | Must be in format fpgax. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
fpga:
enable: @FPGA_ENABLE@
search_devices:
- fpga0
#----------------------+------------------------------------------------------------+------------+-----------------+
# Logs Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
@ -180,12 +180,18 @@ fpga:
# log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 |
# | logging level, num range [0, 1024], 0 means unlimited. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# log_to_stdout | Whether to write logs to standard output in Milvus. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# log_to_file | Whether to write logs to files in Milvus | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
logs:
level: debug
trace.enable: true
path: @MILVUS_DB_PATH@/logs
max_log_file_size: 1024MB
log_rotate_num: 0
log_to_stdout: false
log_to_file: true
#----------------------+------------------------------------------------------------+------------+-----------------+
# Metric Config | Description | Type | Default |

62
core/src/config/Config.cpp
View File

@ -193,6 +193,10 @@ const char* CONFIG_LOGS_LOG_ROTATE_NUM = "log_rotate_num";
const char* CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT = "0";
const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MIN = 0;
const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MAX = 1024;
const char* CONFIG_LOGS_LOG_TO_STDOUT = "log_to_stdout";
const char* CONFIG_LOGS_LOG_TO_STDOUT_DEFAULT = "false";
const char* CONFIG_LOGS_LOG_TO_FILE = "log_to_file";
const char* CONFIG_LOGS_LOG_TO_FILE_DEFAULT = "true";
constexpr int64_t GB = 1UL << 30;
constexpr int32_t PORT_NUMBER_MIN = 1024;
@ -551,6 +555,8 @@ Config::ResetDefaultConfig() {
STATUS_CHECK(SetLogsPath(CONFIG_LOGS_PATH_DEFAULT));
STATUS_CHECK(SetLogsMaxLogFileSize(CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT));
STATUS_CHECK(SetLogsLogRotateNum(CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT));
STATUS_CHECK(SetLogsLogToStdout(CONFIG_LOGS_LOG_TO_STDOUT_DEFAULT));
STATUS_CHECK(SetLogsLogToFile(CONFIG_LOGS_LOG_TO_FILE_DEFAULT));
return Status::OK();
}
@ -720,6 +726,10 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key
status = SetLogsMaxLogFileSize(value);
} else if (child_key == CONFIG_LOGS_LOG_ROTATE_NUM) {
status = SetLogsLogRotateNum(value);
} else if (child_key == CONFIG_LOGS_LOG_TO_STDOUT) {
status = SetLogsLogToStdout(value);
} else if (child_key == CONFIG_LOGS_LOG_TO_FILE) {
status = SetLogsLogToFile(value);
} else {
status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str);
}
@ -1990,6 +2000,30 @@ Config::CheckLogsLogRotateNum(const std::string& value) {
return Status::OK();
}
Status
Config::CheckLogsLogToStdout(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
fiu_do_on("check_logs_log_to_stdout", exist_error = true);
if (exist_error) {
std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.log_to_stdout is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckLogsLogToFile(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
fiu_do_on("check_logs_log_to_file", exist_error = true);
if (exist_error) {
std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.log_to_file is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
////////////////////////////////////////////////////////////////////////////////
ConfigNode&
Config::GetConfigRoot() {
@ -2660,6 +2694,22 @@ Config::GetLogsLogRotateNum(int64_t& value) {
return Status::OK();
}
Status
Config::GetLogsLogToStdout(bool& value) {
std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_LOG_TO_STDOUT, CONFIG_LOGS_LOG_TO_STDOUT_DEFAULT);
STATUS_CHECK(CheckLogsLogToStdout(str));
STATUS_CHECK(StringHelpFunctions::ConvertToBoolean(str, value));
return Status::OK();
}
Status
Config::GetLogsLogToFile(bool& value) {
std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_LOG_TO_FILE, CONFIG_LOGS_LOG_TO_FILE_DEFAULT);
STATUS_CHECK(CheckLogsLogToFile(str));
STATUS_CHECK(StringHelpFunctions::ConvertToBoolean(str, value));
return Status::OK();
}
Status
Config::GetServerRestartRequired(bool& required) {
required = restart_required_;
@ -3035,5 +3085,17 @@ Config::SetLogsLogRotateNum(const std::string& value) {
return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_LOG_ROTATE_NUM, value);
}
Status
Config::SetLogsLogToStdout(const std::string& value) {
STATUS_CHECK(CheckLogsLogToStdout(value));
return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_LOG_TO_STDOUT, value);
}
Status
Config::SetLogsLogToFile(const std::string& value) {
STATUS_CHECK(CheckLogsLogToFile(value));
return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_LOG_TO_FILE, value);
}
} // namespace server
} // namespace milvus

12
core/src/config/Config.h
View File

@ -343,6 +343,10 @@ class Config {
CheckLogsMaxLogFileSize(const std::string& value);
Status
CheckLogsLogRotateNum(const std::string& value);
Status
CheckLogsLogToStdout(const std::string& value);
Status
CheckLogsLogToFile(const std::string& value);
std::string
GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = "");
@ -479,6 +483,10 @@ class Config {
GetLogsMaxLogFileSize(int64_t& value);
Status
GetLogsLogRotateNum(int64_t& value);
Status
GetLogsLogToStdout(bool& value);
Status
GetLogsLogToFile(bool& value);
Status
GetServerRestartRequired(bool& required);
@ -597,6 +605,10 @@ class Config {
SetLogsMaxLogFileSize(const std::string& value);
Status
SetLogsLogRotateNum(const std::string& value);
Status
SetLogsLogToStdout(const std::string& value);
Status
SetLogsLogToFile(const std::string& value);
private:
bool restart_required_ = false;

6
core/src/server/Server.cpp
View File

@ -214,6 +214,8 @@ Server::Start() {
std::string logs_path;
int64_t max_log_file_size = 0;
int64_t delete_exceeds = 0;
bool log_to_stdout = false;
bool log_to_file = true;
STATUS_CHECK(config.GetLogsLevel(level));
switch (level_to_int[level]) {
@ -236,8 +238,10 @@ Server::Start() {
STATUS_CHECK(config.GetLogsPath(logs_path));
STATUS_CHECK(config.GetLogsMaxLogFileSize(max_log_file_size));
STATUS_CHECK(config.GetLogsLogRotateNum(delete_exceeds));
STATUS_CHECK(config.GetLogsLogToStdout(log_to_stdout));
STATUS_CHECK(config.GetLogsLogToFile(log_to_file));
InitLog(trace_enable, debug_enable, info_enable, warning_enable, error_enable, fatal_enable, logs_path,
max_log_file_size, delete_exceeds);
max_log_file_size, delete_exceeds, log_to_stdout, log_to_file);
}
bool cluster_enable = false;

81
core/src/utils/LogUtil.cpp
View File

@ -122,73 +122,68 @@ RolloutHandler(const char* filename, std::size_t size, el::Level level) {
Status
InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_enable, bool error_enable,
bool fatal_enable, const std::string& logs_path, int64_t max_log_file_size, int64_t delete_exceeds) {
bool fatal_enable, const std::string& logs_path, int64_t max_log_file_size, int64_t delete_exceeds,
bool log_to_stdout, bool log_to_file) {
const char* str_true = "true";
const char* str_false = "false";
auto boolen_to_string = [&](bool value) -> const char* { return value ? str_true : str_false; };
el::Configurations defaultConf;
defaultConf.setToDefault();
defaultConf.setGlobally(el::ConfigurationType::Format, "[%datetime][%level]%msg");
defaultConf.setGlobally(el::ConfigurationType::ToFile, "true");
defaultConf.setGlobally(el::ConfigurationType::ToStandardOutput, "false");
defaultConf.setGlobally(el::ConfigurationType::ToFile, boolen_to_string(log_to_file));
defaultConf.setGlobally(el::ConfigurationType::ToStandardOutput, boolen_to_string(log_to_stdout));
defaultConf.setGlobally(el::ConfigurationType::SubsecondPrecision, "3");
defaultConf.setGlobally(el::ConfigurationType::PerformanceTracking, "false");
defaultConf.setGlobally(el::ConfigurationType::PerformanceTracking, str_false);
std::string logs_reg_path = logs_path.rfind('/') == logs_path.length() - 1 ? logs_path : logs_path + "/";
std::string global_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-global.log";
defaultConf.set(el::Level::Global, el::ConfigurationType::Filename, global_log_path.c_str());
defaultConf.set(el::Level::Global, el::ConfigurationType::Enabled, "true");
if (log_to_file) {
std::string global_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-global.log";
defaultConf.set(el::Level::Global, el::ConfigurationType::Filename, global_log_path.c_str());
}
defaultConf.set(el::Level::Global, el::ConfigurationType::Enabled, str_true);
std::string info_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-info.log";
defaultConf.set(el::Level::Info, el::ConfigurationType::Filename, info_log_path.c_str());
fiu_do_on("LogUtil.InitLog.info_enable_to_false", info_enable = false);
if (info_enable) {
defaultConf.set(el::Level::Info, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Info, el::ConfigurationType::Enabled, "false");
if (log_to_file && info_enable) {
std::string info_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-info.log";
defaultConf.set(el::Level::Info, el::ConfigurationType::Filename, info_log_path.c_str());
}
defaultConf.set(el::Level::Info, el::ConfigurationType::Enabled, boolen_to_string(info_enable));
std::string debug_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-debug.log";
defaultConf.set(el::Level::Debug, el::ConfigurationType::Filename, debug_log_path.c_str());
fiu_do_on("LogUtil.InitLog.debug_enable_to_false", debug_enable = false);
if (debug_enable) {
defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, "false");
if (log_to_file && debug_enable) {
std::string debug_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-debug.log";
defaultConf.set(el::Level::Debug, el::ConfigurationType::Filename, debug_log_path.c_str());
}
defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, boolen_to_string(debug_enable));
std::string warning_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-warning.log";
defaultConf.set(el::Level::Warning, el::ConfigurationType::Filename, warning_log_path.c_str());
fiu_do_on("LogUtil.InitLog.warning_enable_to_false", warning_enable = false);
if (warning_enable) {
defaultConf.set(el::Level::Warning, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Warning, el::ConfigurationType::Enabled, "false");
if (log_to_file && warning_enable) {
std::string warning_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-warning.log";
defaultConf.set(el::Level::Warning, el::ConfigurationType::Filename, warning_log_path.c_str());
}
defaultConf.set(el::Level::Warning, el::ConfigurationType::Enabled, boolen_to_string(warning_enable));
std::string trace_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-trace.log";
defaultConf.set(el::Level::Trace, el::ConfigurationType::Filename, trace_log_path.c_str());
fiu_do_on("LogUtil.InitLog.trace_enable_to_false", trace_enable = false);
if (trace_enable) {
defaultConf.set(el::Level::Trace, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Trace, el::ConfigurationType::Enabled, "false");
if (log_to_file && trace_enable) {
std::string trace_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-trace.log";
defaultConf.set(el::Level::Trace, el::ConfigurationType::Filename, trace_log_path.c_str());
}
defaultConf.set(el::Level::Trace, el::ConfigurationType::Enabled, boolen_to_string(trace_enable));
std::string error_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-error.log";
defaultConf.set(el::Level::Error, el::ConfigurationType::Filename, error_log_path.c_str());
fiu_do_on("LogUtil.InitLog.error_enable_to_false", error_enable = false);
if (error_enable) {
defaultConf.set(el::Level::Error, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Error, el::ConfigurationType::Enabled, "false");
if (log_to_file && error_enable) {
std::string error_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-error.log";
defaultConf.set(el::Level::Error, el::ConfigurationType::Filename, error_log_path.c_str());
}
defaultConf.set(el::Level::Error, el::ConfigurationType::Enabled, boolen_to_string(error_enable));
std::string fatal_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log";
defaultConf.set(el::Level::Fatal, el::ConfigurationType::Filename, fatal_log_path.c_str());
fiu_do_on("LogUtil.InitLog.fatal_enable_to_false", fatal_enable = false);
if (fatal_enable) {
defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "false");
if (log_to_file && fatal_enable) {
std::string fatal_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log";
defaultConf.set(el::Level::Fatal, el::ConfigurationType::Filename, fatal_log_path.c_str());
}
defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, boolen_to_string(fatal_enable));
fiu_do_on("LogUtil.InitLog.set_max_log_size_small_than_min",
max_log_file_size = CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN - 1);

3
core/src/utils/LogUtil.h
View File

@ -22,7 +22,8 @@ namespace server {
Status
InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_enable, bool error_enable,
bool fatal_enable, const std::string& logs_path, int64_t max_log_file_size, int64_t delete_exceeds);
bool fatal_enable, const std::string& logs_path, int64_t max_log_file_size, int64_t delete_exceeds,
bool log_to_stdout, bool log_to_file);
void
RolloutHandler(const char* filename, std::size_t size, el::Level level);

14
core/unittest/server/test_config.cpp
View File

@ -443,6 +443,16 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
ASSERT_TRUE(config.SetLogsLogRotateNum(std::to_string(logs_log_rotate_num)).ok());
ASSERT_TRUE(config.GetLogsLogRotateNum(int64_val).ok());
ASSERT_TRUE(int64_val == logs_log_rotate_num);
bool log_to_stdout = true;
ASSERT_TRUE(config.SetLogsLogToStdout(std::to_string(log_to_stdout)).ok());
ASSERT_TRUE(config.GetLogsLogToStdout(bool_val).ok());
ASSERT_TRUE(bool_val == log_to_stdout);
bool log_to_file = false;
ASSERT_TRUE(config.SetLogsLogToFile(std::to_string(log_to_file)).ok());
ASSERT_TRUE(config.GetLogsLogToFile(bool_val).ok());
ASSERT_TRUE(bool_val == log_to_file);
}
std::string
@ -782,7 +792,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
ASSERT_FALSE(config.SetWalConfigBufferSize("-1").ok());
ASSERT_FALSE(config.SetWalConfigBufferSize("a").ok());
/* wal config */
/* log config */
ASSERT_FALSE(config.SetLogsLevel("invalid").ok());
ASSERT_FALSE(config.SetLogsTraceEnable("invalid").ok());
ASSERT_FALSE(config.SetLogsPath("").ok());
@ -790,6 +800,8 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
ASSERT_FALSE(config.SetLogsMaxLogFileSize("511MB").ok());
ASSERT_FALSE(config.SetLogsLogRotateNum("-1").ok());
ASSERT_FALSE(config.SetLogsLogRotateNum("1025").ok());
ASSERT_FALSE(config.SetLogsLogToStdout("invalid").ok());
ASSERT_FALSE(config.SetLogsLogToFile("invalid").ok());
}
TEST_F(ConfigTest, SERVER_CONFIG_TEST) {

8
core/unittest/server/test_util.cpp
View File

@ -250,13 +250,13 @@ TEST(UtilTest, LOG_TEST) {
fiu_enable("LogUtil.InitLog.set_max_log_size_small_than_min", 1, NULL, 0);
auto status = milvus::server::InitLog(true, true, true, true, true, true,
"/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB
"/tmp/test_util", 1024 * 1024 * 1024, 10, false, true); // 1024 MB
ASSERT_FALSE(status.ok());
fiu_disable("LogUtil.InitLog.set_max_log_size_small_than_min");
fiu_enable("LogUtil.InitLog.delete_exceeds_small_than_min", 1, NULL, 0);
status = milvus::server::InitLog(true, true, true, true, true, true,
"/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB
"/tmp/test_util", 1024 * 1024 * 1024, 10, false, true); // 1024 MB
ASSERT_FALSE(status.ok());
fiu_disable("LogUtil.InitLog.delete_exceeds_small_than_min");
@ -267,7 +267,7 @@ TEST(UtilTest, LOG_TEST) {
fiu_enable("LogUtil.InitLog.error_enable_to_false", 1, NULL, 0);
fiu_enable("LogUtil.InitLog.fatal_enable_to_false", 1, NULL, 0);
status = milvus::server::InitLog(true, true, true, true, true, true,
"/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB
"/tmp/test_util", 1024 * 1024 * 1024, 10, false, true); // 1024 MB
ASSERT_TRUE(status.ok()) << status.message();
fiu_disable("LogUtil.InitLog.fatal_enable_to_false");
fiu_disable("LogUtil.InitLog.error_enable_to_false");
@ -277,7 +277,7 @@ TEST(UtilTest, LOG_TEST) {
fiu_disable("LogUtil.InitLog.info_enable_to_false");
status = milvus::server::InitLog(true, true, true, true, true, true,
"/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB
"/tmp/test_util", 1024 * 1024 * 1024, 10, false, true); // 1024 MB
ASSERT_TRUE(status.ok()) << status.message();
EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer));