admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-31 07:55:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

Log file rotating (#2207)

Browse Source 
* log file rotating

Signed-off-by: wxyu <xy.wang@zilliz.com>

* update changelog

Signed-off-by: wxyu <xy.wang@zilliz.com>

* fix compile failed

Signed-off-by: wxyu <xy.wang@zilliz.com>
This commit is contained in:
Wang XiangYu 2020-04-30 23:31:14 +08:00 committed by GitHub
parent 0032c13445
commit ff48427911
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 187 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -27,6 +27,7 @@ Please mark all change in change log and use the issue from GitHub
- \#2059 Add lock file avoid multiple instances modifying data at the same time
- \#2064 Warn when use SQLite as metadata management
- \#2111 Check GPU environment before start server
- \#2206 Log file rotating
## Improvement
- \#221 Refactor LOG macro

2
core/conf/server_config.template
View File

@ -195,4 +195,6 @@ logs:
error.enable: true
fatal.enable: true
path: @MILVUS_DB_PATH@/logs
max_log_file_size: 256
delete_exceeds: 10

88
core/src/config/Config.cpp
View File

@ -169,6 +169,14 @@ const char* CONFIG_LOGS_FATAL_ENABLE = "fatal.enable";
const char* CONFIG_LOGS_FATAL_ENABLE_DEFAULT = "true";
const char* CONFIG_LOGS_PATH = "path";
const char* CONFIG_LOGS_PATH_DEFAULT = "/tmp/milvus/logs";
const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE = "max_log_file_size";
const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT = "256";
const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX = 512;
const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN = 64;
const char* CONFIG_LOGS_DELETE_EXCEEDS = "delete_exceeds";
const char* CONFIG_LOGS_DELETE_EXCEEDS_DEFAULT = "10";
const int64_t CONFIG_LOGS_DELETE_EXCEEDS_MAX = 4096;
const int64_t CONFIG_LOGS_DELETE_EXCEEDS_MIN = 1;
constexpr int64_t GB = 1UL << 30;
constexpr int32_t PORT_NUMBER_MIN = 1024;
@ -406,6 +414,12 @@ Config::ValidateConfig() {
std::string logs_path;
CONFIG_CHECK(GetLogsPath(logs_path));
int64_t logs_max_log_file_size;
CONFIG_CHECK(GetLogsMaxLogFileSize(logs_max_log_file_size));
int64_t delete_exceeds;
CONFIG_CHECK(GetLogsDeleteExceeds(delete_exceeds));
return Status::OK();
}
@ -466,6 +480,8 @@ Config::ResetDefaultConfig() {
CONFIG_CHECK(SetLogsErrorEnable(CONFIG_LOGS_ERROR_ENABLE_DEFAULT));
CONFIG_CHECK(SetLogsFatalEnable(CONFIG_LOGS_FATAL_ENABLE_DEFAULT));
CONFIG_CHECK(SetLogsPath(CONFIG_LOGS_PATH_DEFAULT));
CONFIG_CHECK(SetLogsMaxLogFileSize(CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT));
CONFIG_CHECK(SetLogsDeleteExceeds(CONFIG_LOGS_DELETE_EXCEEDS_DEFAULT));
#ifdef MILVUS_GPU_VERSION
CONFIG_CHECK(SetEngineConfigGpuSearchThreshold(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT));
@ -1625,6 +1641,32 @@ Config::CheckLogsPath(const std::string& value) {
return ValidationUtil::ValidateStoragePath(value);
}
Status
Config::CheckLogsMaxLogFileSize(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();
fiu_do_on("check_logs_max_log_file_size_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid max_log_file_size: " + value +
". Possible reason: logs.max_log_file_size is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckLogsDeleteExceeds(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();
fiu_do_on("check_logs_delete_exceeds_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid max_log_file_size: " + value +
". Possible reason: logs.max_log_file_size is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
////////////////////////////////////////////////////////////////////////////////
ConfigNode&
Config::GetConfigRoot() {
@ -2140,7 +2182,39 @@ Config::GetLogsFatalEnable(bool& value) {
Status
Config::GetLogsPath(std::string& value) {
value = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_PATH, CONFIG_LOGS_PATH_DEFAULT);
CONFIG_CHECK(CheckWalConfigWalPath(value));
CONFIG_CHECK(CheckLogsPath(value));
return Status::OK();
}
Status
Config::GetLogsMaxLogFileSize(int64_t& value) {
std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_MAX_LOG_FILE_SIZE, CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT);
CONFIG_CHECK(CheckLogsMaxLogFileSize(str));
value = std::stoll(str);
if (value == 0) {
// OFF
} else if (value > CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX) {
value = CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX;
} else if (value < CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN) {
value = CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN;
}
return Status::OK();
}
Status
Config::GetLogsDeleteExceeds(int64_t& value) {
std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_DELETE_EXCEEDS, CONFIG_LOGS_DELETE_EXCEEDS_DEFAULT);
CONFIG_CHECK(CheckLogsDeleteExceeds(str));
value = std::stoll(str);
if (value == 0) {
// OFF
} else if (value > CONFIG_LOGS_DELETE_EXCEEDS_MAX) {
value = CONFIG_LOGS_DELETE_EXCEEDS_MAX;
} else if (value < CONFIG_LOGS_DELETE_EXCEEDS_MIN) {
value = CONFIG_LOGS_DELETE_EXCEEDS_MIN;
}
return Status::OK();
}
@ -2411,6 +2485,18 @@ Config::SetLogsPath(const std::string& value) {
return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_PATH, value);
}
Status
Config::SetLogsMaxLogFileSize(const std::string& value) {
CONFIG_CHECK(CheckLogsMaxLogFileSize(value));
return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_MAX_LOG_FILE_SIZE, value);
}
Status
Config::SetLogsDeleteExceeds(const std::string& value) {
CONFIG_CHECK(CheckLogsDeleteExceeds(value));
return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_DELETE_EXCEEDS, value);
}
#ifdef MILVUS_GPU_VERSION
Status
Config::SetEngineConfigGpuSearchThreshold(const std::string& value) {

20
core/src/config/Config.h
View File

@ -164,6 +164,14 @@ extern const char* CONFIG_LOGS_ERROR_ENABLE_DEFAULT;
extern const char* CONFIG_LOGS_FATAL_ENABLE;
extern const char* CONFIG_LOGS_FATAL_ENABLE_DEFAULT;
extern const char* CONFIG_LOGS_PATH;
extern const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE;
extern const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT;
extern const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX;
extern const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN;
extern const char* CONFIG_LOGS_DELETE_EXCEEDS;
extern const char* CONFIG_LOGS_DELETE_EXCEEDS_DEFAULT;
extern const int64_t CONFIG_LOGS_DELETE_EXCEEDS_MAX;
extern const int64_t CONFIG_LOGS_DELETE_EXCEEDS_MIN;
class Config {
private:
@ -332,6 +340,10 @@ class Config {
CheckLogsFatalEnable(const std::string& value);
Status
CheckLogsPath(const std::string& value);
Status
CheckLogsMaxLogFileSize(const std::string& value);
Status
CheckLogsDeleteExceeds(const std::string& value);
std::string
GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = "");
@ -461,6 +473,10 @@ class Config {
GetLogsFatalEnable(bool& value);
Status
GetLogsPath(std::string& value);
Status
GetLogsMaxLogFileSize(int64_t& value);
Status
GetLogsDeleteExceeds(int64_t& value);
Status
GetServerRestartRequired(bool& required);
@ -565,6 +581,10 @@ class Config {
SetLogsFatalEnable(const std::string& value);
Status
SetLogsPath(const std::string& value);
Status
SetLogsMaxLogFileSize(const std::string& value);
Status
SetLogsDeleteExceeds(const std::string& value);
#ifdef MILVUS_GPU_VERSION
Status

13
core/src/server/Server.cpp
View File

@ -199,6 +199,8 @@ Server::Start() {
bool error_enable = false;
bool fatal_enable = false;
std::string logs_path;
int64_t max_log_file_size = 0;
int64_t delete_exceeds = 0;
s = config.GetLogsTraceEnable(trace_enable);
if (!s.ok()) {
return s;
@ -227,7 +229,16 @@ Server::Start() {
if (!s.ok()) {
return s;
}
InitLog(trace_enable, debug_enable, info_enable, warning_enable, error_enable, fatal_enable, logs_path);
s = config.GetLogsMaxLogFileSize(max_log_file_size);
if (!s.ok()) {
return s;
}
s = config.GetLogsDeleteExceeds(delete_exceeds);
if (!s.ok()) {
return s;
}
InitLog(trace_enable, debug_enable, info_enable, warning_enable, error_enable, fatal_enable, logs_path,
max_log_file_size, delete_exceeds);
}
std::string deploy_mode;

75
core/src/utils/LogUtil.cpp
View File

@ -16,6 +16,7 @@
#include <string>
#include <yaml-cpp/yaml.h>
#include <boost/filesystem.hpp>
#include "config/Config.h"
#include "utils/Log.h"
@ -30,6 +31,7 @@ static int warning_idx = 0;
static int trace_idx = 0;
static int error_idx = 0;
static int fatal_idx = 0;
static int64_t logs_delete_exceeds = 1;
} // namespace
// TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename
@ -55,36 +57,68 @@ RolloutHandler(const char* filename, std::size_t size, el::Level level) {
std::string m(std::string(dir) + "/" + s);
s = m;
switch (level) {
case el::Level::Debug:
case el::Level::Debug: {
s.append("." + std::to_string(++debug_idx));
ret = rename(m.c_str(), s.c_str());
// std::cout << "debug_idx:" << debug_idx << ", logs_delete_exceeds:" << logs_delete_exceeds << std::endl;
if (debug_idx - logs_delete_exceeds > 0) {
std::string to_delete = m + "." + std::to_string(debug_idx - logs_delete_exceeds);
// std::cout << "remote " << to_delete << std::endl;
boost::filesystem::remove(to_delete);
}
break;
case el::Level::Warning:
}
case el::Level::Warning: {
s.append("." + std::to_string(++warning_idx));
ret = rename(m.c_str(), s.c_str());
if (warning_idx - logs_delete_exceeds > 0) {
std::string to_delete = m + "." + std::to_string(warning_idx - logs_delete_exceeds);
boost::filesystem::remove(to_delete);
}
break;
case el::Level::Trace:
}
case el::Level::Trace: {
s.append("." + std::to_string(++trace_idx));
ret = rename(m.c_str(), s.c_str());
if (trace_idx - logs_delete_exceeds > 0) {
std::string to_delete = m + "." + std::to_string(trace_idx - logs_delete_exceeds);
boost::filesystem::remove(to_delete);
}
break;
case el::Level::Error:
}
case el::Level::Error: {
s.append("." + std::to_string(++error_idx));
ret = rename(m.c_str(), s.c_str());
if (error_idx - logs_delete_exceeds > 0) {
std::string to_delete = m + "." + std::to_string(error_idx - logs_delete_exceeds);
boost::filesystem::remove(to_delete);
}
break;
case el::Level::Fatal:
}
case el::Level::Fatal: {
s.append("." + std::to_string(++fatal_idx));
ret = rename(m.c_str(), s.c_str());
if (fatal_idx - logs_delete_exceeds > 0) {
std::string to_delete = m + "." + std::to_string(fatal_idx - logs_delete_exceeds);
boost::filesystem::remove(to_delete);
}
break;
default:
}
default: {
s.append("." + std::to_string(++global_idx));
ret = rename(m.c_str(), s.c_str());
if (global_idx - logs_delete_exceeds > 0) {
std::string to_delete = m + "." + std::to_string(global_idx - logs_delete_exceeds);
boost::filesystem::remove(to_delete);
}
break;
}
}
}
Status
InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_enable, bool error_enable,
bool fatal_enable, const std::string& logs_path) {
bool fatal_enable, const std::string& logs_path, int64_t max_log_file_size, int64_t delete_exceeds) {
el::Configurations defaultConf;
defaultConf.setToDefault();
defaultConf.setGlobally(el::ConfigurationType::Format, "[%datetime][%level]%msg");
@ -92,7 +126,6 @@ InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_ena
defaultConf.setGlobally(el::ConfigurationType::ToStandardOutput, "false");
defaultConf.setGlobally(el::ConfigurationType::SubsecondPrecision, "3");
defaultConf.setGlobally(el::ConfigurationType::PerformanceTracking, "false");
defaultConf.setGlobally(el::ConfigurationType::MaxLogFileSize, "209715200");
std::string logs_reg_path = logs_path.rfind('/') == logs_path.length() - 1 ? logs_path : logs_path + "/";
std::string global_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-global.log";
@ -147,11 +180,29 @@ InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_ena
defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "false");
}
el::Loggers::reconfigureLogger("default", defaultConf);
// set max_log_file_size = 0 means disable log file rotating
if (max_log_file_size != 0) {
if (max_log_file_size < 64 || max_log_file_size > 512) {
return Status(SERVER_UNEXPECTED_ERROR,
"max_log_file_size must in range[64, 512], now is " + std::to_string(max_log_file_size));
}
max_log_file_size *= 1024 * 1024;
defaultConf.setGlobally(el::ConfigurationType::MaxLogFileSize, std::to_string(max_log_file_size));
el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck);
el::Helpers::installPreRollOutCallback(RolloutHandler);
el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog);
el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck);
el::Helpers::installPreRollOutCallback(RolloutHandler);
el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog);
// set delete_exceeds = 0 means disable throw away log file even they reach certain limit.
if (delete_exceeds != 0) {
if (delete_exceeds < 1 || delete_exceeds > 4096) {
return Status(SERVER_UNEXPECTED_ERROR,
"delete_exceeds must in range[1, 4096], now is " + std::to_string(delete_exceeds));
}
logs_delete_exceeds = delete_exceeds;
}
}
el::Loggers::reconfigureLogger("default", defaultConf);
return Status::OK();
}

2
core/src/utils/LogUtil.h
View File

@ -22,7 +22,7 @@ namespace server {
Status
InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_enable, bool error_enable,
bool fatal_enable, const std::string& logs_path);
bool fatal_enable, const std::string& logs_path, int64_t max_log_file_size, int64_t delete_exceeds);
void
RolloutHandler(const char* filename, std::size_t size, el::Level level);

2
core/unittest/server/test_util.cpp
View File

@ -246,7 +246,7 @@ TEST(UtilTest, BLOCKINGQUEUE_TEST) {
}
TEST(UtilTest, LOG_TEST) {
auto status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util");
auto status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util", 256, 10);
ASSERT_TRUE(status.ok());
EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer));