diff --git a/CHANGELOG.md b/CHANGELOG.md index a813b91a65..910431927f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,21 @@ Please mark all change in change log and use the issue from GitHub -# Milvus 0.10.6 (TBD) +# Milvus 1.0.0-beta (TBD) +## Bug + +## Feature +- \#3977 Support logging to stdout + +## Improvement + +## Task + +# Milvus 0.10.6 (2021-02-23) ## Bug - \#4683 A negative zero may be returned if the metric_type is Tanimoto - \#4678 Server crash on BinaryFlat if dimension is not a power of 2 -- \#4720 The GPU cache holds much more data than the configured value +- \#4719 The GPU cache holds much more data than the configured value ## Feature diff --git a/core/conf/demo/server_config.yaml b/core/conf/demo/server_config.yaml index 2b24f339ff..3ba3f364e1 100644 --- a/core/conf/demo/server_config.yaml +++ b/core/conf/demo/server_config.yaml @@ -179,12 +179,18 @@ fpga: # log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 | # | logging level, num range [0, 1024], 0 means unlimited. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ +# log_to_stdout | Whether to write logs to standard output in Milvus. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# log_to_file | Whether to write logs to files in Milvus | Boolean | true | +#----------------------+------------------------------------------------------------+------------+-----------------+ logs: level: debug trace.enable: true path: /var/lib/milvus/logs max_log_file_size: 1024MB log_rotate_num: 0 + log_to_stdout: false + log_to_file: true #----------------------+------------------------------------------------------------+------------+-----------------+ # Metric Config | Description | Type | Default | diff --git a/core/conf/server_config.template b/core/conf/server_config.template index 29c7b83193..6be68deaa6 100644 --- a/core/conf/server_config.template +++ b/core/conf/server_config.template @@ -159,11 +159,11 @@ gpu: # search_devices | The list of FPGA devices used for search computation. | DeviceList | fpga0 | # | Must be in format fpgax. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ - fpga: enable: @FPGA_ENABLE@ search_devices: - fpga0 + #----------------------+------------------------------------------------------------+------------+-----------------+ # Logs Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -180,12 +180,18 @@ fpga: # log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 | # | logging level, num range [0, 1024], 0 means unlimited. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ +# log_to_stdout | Whether to write logs to standard output in Milvus. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# log_to_file | Whether to write logs to files in Milvus | Boolean | true | +#----------------------+------------------------------------------------------------+------------+-----------------+ logs: level: debug trace.enable: true path: @MILVUS_DB_PATH@/logs max_log_file_size: 1024MB log_rotate_num: 0 + log_to_stdout: false + log_to_file: true #----------------------+------------------------------------------------------------+------------+-----------------+ # Metric Config | Description | Type | Default | diff --git a/core/src/config/Config.cpp b/core/src/config/Config.cpp index a224fdf7dd..24e7ebd03c 100644 --- a/core/src/config/Config.cpp +++ b/core/src/config/Config.cpp @@ -193,6 +193,10 @@ const char* CONFIG_LOGS_LOG_ROTATE_NUM = "log_rotate_num"; const char* CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT = "0"; const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MIN = 0; const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MAX = 1024; +const char* CONFIG_LOGS_LOG_TO_STDOUT = "log_to_stdout"; +const char* CONFIG_LOGS_LOG_TO_STDOUT_DEFAULT = "false"; +const char* CONFIG_LOGS_LOG_TO_FILE = "log_to_file"; +const char* CONFIG_LOGS_LOG_TO_FILE_DEFAULT = "true"; constexpr int64_t GB = 1UL << 30; constexpr int32_t PORT_NUMBER_MIN = 1024; @@ -551,6 +555,8 @@ Config::ResetDefaultConfig() { STATUS_CHECK(SetLogsPath(CONFIG_LOGS_PATH_DEFAULT)); STATUS_CHECK(SetLogsMaxLogFileSize(CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT)); STATUS_CHECK(SetLogsLogRotateNum(CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT)); + STATUS_CHECK(SetLogsLogToStdout(CONFIG_LOGS_LOG_TO_STDOUT_DEFAULT)); + STATUS_CHECK(SetLogsLogToFile(CONFIG_LOGS_LOG_TO_FILE_DEFAULT)); return Status::OK(); } @@ -720,6 +726,10 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key status = SetLogsMaxLogFileSize(value); } else if (child_key == CONFIG_LOGS_LOG_ROTATE_NUM) { status = SetLogsLogRotateNum(value); + } else if (child_key == CONFIG_LOGS_LOG_TO_STDOUT) { + status = SetLogsLogToStdout(value); + } else if (child_key == CONFIG_LOGS_LOG_TO_FILE) { + status = SetLogsLogToFile(value); } else { status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } @@ -1990,6 +2000,30 @@ Config::CheckLogsLogRotateNum(const std::string& value) { return Status::OK(); } +Status +Config::CheckLogsLogToStdout(const std::string& value) { + auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok(); + fiu_do_on("check_logs_log_to_stdout", exist_error = true); + + if (exist_error) { + std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.log_to_stdout is not a boolean."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + return Status::OK(); +} + +Status +Config::CheckLogsLogToFile(const std::string& value) { + auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok(); + fiu_do_on("check_logs_log_to_file", exist_error = true); + + if (exist_error) { + std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.log_to_file is not a boolean."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + return Status::OK(); +} + //////////////////////////////////////////////////////////////////////////////// ConfigNode& Config::GetConfigRoot() { @@ -2660,6 +2694,22 @@ Config::GetLogsLogRotateNum(int64_t& value) { return Status::OK(); } +Status +Config::GetLogsLogToStdout(bool& value) { + std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_LOG_TO_STDOUT, CONFIG_LOGS_LOG_TO_STDOUT_DEFAULT); + STATUS_CHECK(CheckLogsLogToStdout(str)); + STATUS_CHECK(StringHelpFunctions::ConvertToBoolean(str, value)); + return Status::OK(); +} + +Status +Config::GetLogsLogToFile(bool& value) { + std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_LOG_TO_FILE, CONFIG_LOGS_LOG_TO_FILE_DEFAULT); + STATUS_CHECK(CheckLogsLogToFile(str)); + STATUS_CHECK(StringHelpFunctions::ConvertToBoolean(str, value)); + return Status::OK(); +} + Status Config::GetServerRestartRequired(bool& required) { required = restart_required_; @@ -3035,5 +3085,17 @@ Config::SetLogsLogRotateNum(const std::string& value) { return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_LOG_ROTATE_NUM, value); } +Status +Config::SetLogsLogToStdout(const std::string& value) { + STATUS_CHECK(CheckLogsLogToStdout(value)); + return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_LOG_TO_STDOUT, value); +} + +Status +Config::SetLogsLogToFile(const std::string& value) { + STATUS_CHECK(CheckLogsLogToFile(value)); + return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_LOG_TO_FILE, value); +} + } // namespace server } // namespace milvus diff --git a/core/src/config/Config.h b/core/src/config/Config.h index 0caadc8bf3..10a191582b 100644 --- a/core/src/config/Config.h +++ b/core/src/config/Config.h @@ -343,6 +343,10 @@ class Config { CheckLogsMaxLogFileSize(const std::string& value); Status CheckLogsLogRotateNum(const std::string& value); + Status + CheckLogsLogToStdout(const std::string& value); + Status + CheckLogsLogToFile(const std::string& value); std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); @@ -479,6 +483,10 @@ class Config { GetLogsMaxLogFileSize(int64_t& value); Status GetLogsLogRotateNum(int64_t& value); + Status + GetLogsLogToStdout(bool& value); + Status + GetLogsLogToFile(bool& value); Status GetServerRestartRequired(bool& required); @@ -597,6 +605,10 @@ class Config { SetLogsMaxLogFileSize(const std::string& value); Status SetLogsLogRotateNum(const std::string& value); + Status + SetLogsLogToStdout(const std::string& value); + Status + SetLogsLogToFile(const std::string& value); private: bool restart_required_ = false; diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index fcdd0a7624..f9f83d84e8 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -214,6 +214,8 @@ Server::Start() { std::string logs_path; int64_t max_log_file_size = 0; int64_t delete_exceeds = 0; + bool log_to_stdout = false; + bool log_to_file = true; STATUS_CHECK(config.GetLogsLevel(level)); switch (level_to_int[level]) { @@ -236,8 +238,10 @@ Server::Start() { STATUS_CHECK(config.GetLogsPath(logs_path)); STATUS_CHECK(config.GetLogsMaxLogFileSize(max_log_file_size)); STATUS_CHECK(config.GetLogsLogRotateNum(delete_exceeds)); + STATUS_CHECK(config.GetLogsLogToStdout(log_to_stdout)); + STATUS_CHECK(config.GetLogsLogToFile(log_to_file)); InitLog(trace_enable, debug_enable, info_enable, warning_enable, error_enable, fatal_enable, logs_path, - max_log_file_size, delete_exceeds); + max_log_file_size, delete_exceeds, log_to_stdout, log_to_file); } bool cluster_enable = false; diff --git a/core/src/utils/LogUtil.cpp b/core/src/utils/LogUtil.cpp index 1491c73852..1443120751 100644 --- a/core/src/utils/LogUtil.cpp +++ b/core/src/utils/LogUtil.cpp @@ -122,73 +122,68 @@ RolloutHandler(const char* filename, std::size_t size, el::Level level) { Status InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_enable, bool error_enable, - bool fatal_enable, const std::string& logs_path, int64_t max_log_file_size, int64_t delete_exceeds) { + bool fatal_enable, const std::string& logs_path, int64_t max_log_file_size, int64_t delete_exceeds, + bool log_to_stdout, bool log_to_file) { + const char* str_true = "true"; + const char* str_false = "false"; + auto boolen_to_string = [&](bool value) -> const char* { return value ? str_true : str_false; }; + el::Configurations defaultConf; defaultConf.setToDefault(); defaultConf.setGlobally(el::ConfigurationType::Format, "[%datetime][%level]%msg"); - defaultConf.setGlobally(el::ConfigurationType::ToFile, "true"); - defaultConf.setGlobally(el::ConfigurationType::ToStandardOutput, "false"); + defaultConf.setGlobally(el::ConfigurationType::ToFile, boolen_to_string(log_to_file)); + defaultConf.setGlobally(el::ConfigurationType::ToStandardOutput, boolen_to_string(log_to_stdout)); defaultConf.setGlobally(el::ConfigurationType::SubsecondPrecision, "3"); - defaultConf.setGlobally(el::ConfigurationType::PerformanceTracking, "false"); + defaultConf.setGlobally(el::ConfigurationType::PerformanceTracking, str_false); std::string logs_reg_path = logs_path.rfind('/') == logs_path.length() - 1 ? logs_path : logs_path + "/"; - std::string global_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-global.log"; - defaultConf.set(el::Level::Global, el::ConfigurationType::Filename, global_log_path.c_str()); - defaultConf.set(el::Level::Global, el::ConfigurationType::Enabled, "true"); + if (log_to_file) { + std::string global_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-global.log"; + defaultConf.set(el::Level::Global, el::ConfigurationType::Filename, global_log_path.c_str()); + } + defaultConf.set(el::Level::Global, el::ConfigurationType::Enabled, str_true); - std::string info_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-info.log"; - defaultConf.set(el::Level::Info, el::ConfigurationType::Filename, info_log_path.c_str()); fiu_do_on("LogUtil.InitLog.info_enable_to_false", info_enable = false); - if (info_enable) { - defaultConf.set(el::Level::Info, el::ConfigurationType::Enabled, "true"); - } else { - defaultConf.set(el::Level::Info, el::ConfigurationType::Enabled, "false"); + if (log_to_file && info_enable) { + std::string info_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-info.log"; + defaultConf.set(el::Level::Info, el::ConfigurationType::Filename, info_log_path.c_str()); } + defaultConf.set(el::Level::Info, el::ConfigurationType::Enabled, boolen_to_string(info_enable)); - std::string debug_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-debug.log"; - defaultConf.set(el::Level::Debug, el::ConfigurationType::Filename, debug_log_path.c_str()); fiu_do_on("LogUtil.InitLog.debug_enable_to_false", debug_enable = false); - if (debug_enable) { - defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, "true"); - } else { - defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, "false"); + if (log_to_file && debug_enable) { + std::string debug_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-debug.log"; + defaultConf.set(el::Level::Debug, el::ConfigurationType::Filename, debug_log_path.c_str()); } + defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, boolen_to_string(debug_enable)); - std::string warning_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-warning.log"; - defaultConf.set(el::Level::Warning, el::ConfigurationType::Filename, warning_log_path.c_str()); fiu_do_on("LogUtil.InitLog.warning_enable_to_false", warning_enable = false); - if (warning_enable) { - defaultConf.set(el::Level::Warning, el::ConfigurationType::Enabled, "true"); - } else { - defaultConf.set(el::Level::Warning, el::ConfigurationType::Enabled, "false"); + if (log_to_file && warning_enable) { + std::string warning_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-warning.log"; + defaultConf.set(el::Level::Warning, el::ConfigurationType::Filename, warning_log_path.c_str()); } + defaultConf.set(el::Level::Warning, el::ConfigurationType::Enabled, boolen_to_string(warning_enable)); - std::string trace_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-trace.log"; - defaultConf.set(el::Level::Trace, el::ConfigurationType::Filename, trace_log_path.c_str()); fiu_do_on("LogUtil.InitLog.trace_enable_to_false", trace_enable = false); - if (trace_enable) { - defaultConf.set(el::Level::Trace, el::ConfigurationType::Enabled, "true"); - } else { - defaultConf.set(el::Level::Trace, el::ConfigurationType::Enabled, "false"); + if (log_to_file && trace_enable) { + std::string trace_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-trace.log"; + defaultConf.set(el::Level::Trace, el::ConfigurationType::Filename, trace_log_path.c_str()); } + defaultConf.set(el::Level::Trace, el::ConfigurationType::Enabled, boolen_to_string(trace_enable)); - std::string error_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-error.log"; - defaultConf.set(el::Level::Error, el::ConfigurationType::Filename, error_log_path.c_str()); fiu_do_on("LogUtil.InitLog.error_enable_to_false", error_enable = false); - if (error_enable) { - defaultConf.set(el::Level::Error, el::ConfigurationType::Enabled, "true"); - } else { - defaultConf.set(el::Level::Error, el::ConfigurationType::Enabled, "false"); + if (log_to_file && error_enable) { + std::string error_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-error.log"; + defaultConf.set(el::Level::Error, el::ConfigurationType::Filename, error_log_path.c_str()); } + defaultConf.set(el::Level::Error, el::ConfigurationType::Enabled, boolen_to_string(error_enable)); - std::string fatal_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log"; - defaultConf.set(el::Level::Fatal, el::ConfigurationType::Filename, fatal_log_path.c_str()); fiu_do_on("LogUtil.InitLog.fatal_enable_to_false", fatal_enable = false); - if (fatal_enable) { - defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "true"); - } else { - defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "false"); + if (log_to_file && fatal_enable) { + std::string fatal_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log"; + defaultConf.set(el::Level::Fatal, el::ConfigurationType::Filename, fatal_log_path.c_str()); } + defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, boolen_to_string(fatal_enable)); fiu_do_on("LogUtil.InitLog.set_max_log_size_small_than_min", max_log_file_size = CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN - 1); diff --git a/core/src/utils/LogUtil.h b/core/src/utils/LogUtil.h index 7c9b2533cb..e4a469d347 100644 --- a/core/src/utils/LogUtil.h +++ b/core/src/utils/LogUtil.h @@ -22,7 +22,8 @@ namespace server { Status InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_enable, bool error_enable, - bool fatal_enable, const std::string& logs_path, int64_t max_log_file_size, int64_t delete_exceeds); + bool fatal_enable, const std::string& logs_path, int64_t max_log_file_size, int64_t delete_exceeds, + bool log_to_stdout, bool log_to_file); void RolloutHandler(const char* filename, std::size_t size, el::Level level); diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index f665cded97..c49b818d97 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -443,6 +443,16 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(config.SetLogsLogRotateNum(std::to_string(logs_log_rotate_num)).ok()); ASSERT_TRUE(config.GetLogsLogRotateNum(int64_val).ok()); ASSERT_TRUE(int64_val == logs_log_rotate_num); + + bool log_to_stdout = true; + ASSERT_TRUE(config.SetLogsLogToStdout(std::to_string(log_to_stdout)).ok()); + ASSERT_TRUE(config.GetLogsLogToStdout(bool_val).ok()); + ASSERT_TRUE(bool_val == log_to_stdout); + + bool log_to_file = false; + ASSERT_TRUE(config.SetLogsLogToFile(std::to_string(log_to_file)).ok()); + ASSERT_TRUE(config.GetLogsLogToFile(bool_val).ok()); + ASSERT_TRUE(bool_val == log_to_file); } std::string @@ -782,7 +792,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(config.SetWalConfigBufferSize("-1").ok()); ASSERT_FALSE(config.SetWalConfigBufferSize("a").ok()); - /* wal config */ + /* log config */ ASSERT_FALSE(config.SetLogsLevel("invalid").ok()); ASSERT_FALSE(config.SetLogsTraceEnable("invalid").ok()); ASSERT_FALSE(config.SetLogsPath("").ok()); @@ -790,6 +800,8 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(config.SetLogsMaxLogFileSize("511MB").ok()); ASSERT_FALSE(config.SetLogsLogRotateNum("-1").ok()); ASSERT_FALSE(config.SetLogsLogRotateNum("1025").ok()); + ASSERT_FALSE(config.SetLogsLogToStdout("invalid").ok()); + ASSERT_FALSE(config.SetLogsLogToFile("invalid").ok()); } TEST_F(ConfigTest, SERVER_CONFIG_TEST) { diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index 1edd98bf0d..0fdd3d5040 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -250,13 +250,13 @@ TEST(UtilTest, LOG_TEST) { fiu_enable("LogUtil.InitLog.set_max_log_size_small_than_min", 1, NULL, 0); auto status = milvus::server::InitLog(true, true, true, true, true, true, - "/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB + "/tmp/test_util", 1024 * 1024 * 1024, 10, false, true); // 1024 MB ASSERT_FALSE(status.ok()); fiu_disable("LogUtil.InitLog.set_max_log_size_small_than_min"); fiu_enable("LogUtil.InitLog.delete_exceeds_small_than_min", 1, NULL, 0); status = milvus::server::InitLog(true, true, true, true, true, true, - "/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB + "/tmp/test_util", 1024 * 1024 * 1024, 10, false, true); // 1024 MB ASSERT_FALSE(status.ok()); fiu_disable("LogUtil.InitLog.delete_exceeds_small_than_min"); @@ -267,7 +267,7 @@ TEST(UtilTest, LOG_TEST) { fiu_enable("LogUtil.InitLog.error_enable_to_false", 1, NULL, 0); fiu_enable("LogUtil.InitLog.fatal_enable_to_false", 1, NULL, 0); status = milvus::server::InitLog(true, true, true, true, true, true, - "/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB + "/tmp/test_util", 1024 * 1024 * 1024, 10, false, true); // 1024 MB ASSERT_TRUE(status.ok()) << status.message(); fiu_disable("LogUtil.InitLog.fatal_enable_to_false"); fiu_disable("LogUtil.InitLog.error_enable_to_false"); @@ -277,7 +277,7 @@ TEST(UtilTest, LOG_TEST) { fiu_disable("LogUtil.InitLog.info_enable_to_false"); status = milvus::server::InitLog(true, true, true, true, true, true, - "/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB + "/tmp/test_util", 1024 * 1024 * 1024, 10, false, true); // 1024 MB ASSERT_TRUE(status.ok()) << status.message(); EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer));