From a641bb4bb3b1139916ae634610cb477f979e425c Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Mon, 9 Sep 2019 19:15:27 +0800 Subject: [PATCH 1/3] MS-119 The problem of combining the log files Former-commit-id: b3ade33ab3348ef3108c60faa62e8e3239ad5a00 --- cpp/CHANGELOG.md | 1 + cpp/src/utils/LogUtil.cpp | 49 ++++++++++++++++++++++++++++++++++++++- cpp/src/utils/LogUtil.h | 9 +++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 3d87261868..742c64fe7d 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -29,6 +29,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-492 - Drop index failed if index have been created with index_type: FLAT - MS-493 - Knowhere unittest crash - MS-453 - GPU search error when nprobe set more than 1024 +- MS-119 - The problem of combining the log files ## Improvement - MS-327 - Clean code for milvus diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index 07f3b92aa5..608a7e7408 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -9,6 +9,9 @@ #include #include +#include +#include + namespace zilliz { namespace milvus { namespace server { @@ -50,11 +53,55 @@ int32_t InitLog(const std::string& log_config_file) { #else el::Configurations conf(log_config_file); #endif - el::Loggers::reconfigureAllLoggers(conf); + + el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck); + el::Helpers::installPreRollOutCallback(rolloutHandler); return 0; } +// TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename +void rolloutHandler(const char* filename, std::size_t size){ + char *dirc = strdup(filename); + char *basec = strdup(filename); + char *dir = dirname(dirc); + char *base = basename(basec); + + std::string s(base); + std::stringstream ss; + std::string list[] = {"\\", " ", "\'", "\"", "*", "\?", "{", "}", ";", "<", ">", "|", "^", "&", "$", "#", "!", "`", "~"}; + std::string::size_type position; + for(auto substr : list){ + position = 0; + while((position = s.find_first_of(substr, position)) != std::string::npos){ + s.insert(position, "\\"); + position += 2; + } + } + if((position = s.find("global")) != std::string::npos){ + ss << "mv " << dir << "/" << s << " " << dir << "/" << s << "." << ++global_idx; + } + else if((position = s.find("debug")) != std::string::npos){ + ss << "mv " << dir << "/" << s << " " << dir << "/" << s << "." << ++debug_idx; + } + else if((position = s.find("warning")) != std::string::npos){ + ss << "mv " << dir << "/" << s << " " << dir << "/" << s << "." << ++warning_idx; + } + else if((position = s.find("trace")) != std::string::npos){ + ss << "mv " << dir << "/" << s << " " << dir << "/" << s << "." << ++trace_idx; + } + else if((position = s.find("error")) != std::string::npos){ + ss << "mv " << dir << "/" << s << " " << dir << "/" << s << "." << ++error_idx; + } + else if((position = s.find("fatal")) != std::string::npos){ + ss << "mv " << dir << "/" << s << " " << dir << "/" << s << "." << ++fatal_idx; + } + else{ + ss << "mv " << dir << "/" << s << " " << dir << "/" << s << "." << ++global_idx; + } +// std::cout << ss.str() << std::endl; + system(ss.str().c_str()); +} } // server } // milvus diff --git a/cpp/src/utils/LogUtil.h b/cpp/src/utils/LogUtil.h index 4a5b463c42..0d98e8133a 100644 --- a/cpp/src/utils/LogUtil.h +++ b/cpp/src/utils/LogUtil.h @@ -12,8 +12,17 @@ namespace zilliz { namespace milvus { namespace server { +static unsigned global_idx = 0; +static unsigned debug_idx = 0; +static unsigned warning_idx = 0; +static unsigned trace_idx = 0; +static unsigned error_idx = 0; +static unsigned fatal_idx = 0; + int32_t InitLog(const std::string& log_config_file); +void rolloutHandler(const char* filename, std::size_t size); + inline std::string GetFileName(std::string filename) { int pos = filename.find_last_of('/'); return filename.substr(pos + 1); From c54247c247adeb875e8040a0a9aeba6a840566b3 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Mon, 9 Sep 2019 20:27:10 +0800 Subject: [PATCH 2/3] change the code under suggestions Former-commit-id: e4bb750aa14b6405c6af434239f71037a7de2744 --- cpp/conf/log_config.template | 2 +- cpp/src/utils/LogUtil.cpp | 100 ++++++++++++++++++++--------------- cpp/src/utils/LogUtil.h | 11 ---- 3 files changed, 58 insertions(+), 55 deletions(-) diff --git a/cpp/conf/log_config.template b/cpp/conf/log_config.template index cc33899589..e4b0841d0b 100644 --- a/cpp/conf/log_config.template +++ b/cpp/conf/log_config.template @@ -6,7 +6,7 @@ TO_STANDARD_OUTPUT = false SUBSECOND_PRECISION = 3 PERFORMANCE_TRACKING = false - MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB + MAX_LOG_FILE_SIZE = 10240 ##209715200 ## Throw log files away after 200MB * DEBUG: FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-debug.log" ENABLED = true diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index 608a7e7408..ec6f078500 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -12,11 +12,67 @@ #include #include + namespace zilliz { namespace milvus { namespace server { -int32_t InitLog(const std::string& log_config_file) { +namespace { +static int global_idx = 0; +static int debug_idx = 0; +static int warning_idx = 0; +static int trace_idx = 0; +static int error_idx = 0; +static int fatal_idx = 0; +} + +// TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename +void rolloutHandler(const char *filename, std::size_t size) { + char *dirc = strdup(filename); + char *basec = strdup(filename); + char *dir = dirname(dirc); + char *base = basename(basec); + + std::string s(base); + std::stringstream ss; + std::string + list[] = {"\\", " ", "\'", "\"", "*", "\?", "{", "}", ";", "<", ">", "|", "^", "&", "$", "#", "!", "`", "~"}; + std::string::size_type position; + for (auto substr : list) { + position = 0; + while ((position = s.find_first_of(substr, position)) != std::string::npos) { + s.insert(position, "\\"); + position += 2; + } + } + int ret; + std::string m(std::string(dir) + "/" + s); + s = m; + if ((position = s.find("global")) != std::string::npos) { + s.append("." + std::to_string(++global_idx)); + ret = rename(m.c_str(), s.c_str()); + } else if ((position = s.find("debug")) != std::string::npos) { + s.append("." + std::to_string(++debug_idx)); + ret = rename(m.c_str(), s.c_str()); + } else if ((position = s.find("warning")) != std::string::npos) { + s.append("." + std::to_string(++warning_idx)); + ret = rename(m.c_str(), s.c_str()); + } else if ((position = s.find("trace")) != std::string::npos) { + s.append("." + std::to_string(++trace_idx)); + ret = rename(m.c_str(), s.c_str()); + } else if ((position = s.find("error")) != std::string::npos) { + s.append("." + std::to_string(++error_idx)); + ret = rename(m.c_str(), s.c_str()); + } else if ((position = s.find("fatal")) != std::string::npos) { + s.append("." + std::to_string(++fatal_idx)); + ret = rename(m.c_str(), s.c_str()); + } else { + s.append("." + std::to_string(++global_idx)); + ret = rename(m.c_str(), s.c_str()); + } +} + +int32_t InitLog(const std::string &log_config_file) { #if 0 ServerConfig &config = ServerConfig::GetInstance(); ConfigNode log_config = config.GetConfig(CONFIG_LOG); @@ -60,48 +116,6 @@ int32_t InitLog(const std::string& log_config_file) { return 0; } -// TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename -void rolloutHandler(const char* filename, std::size_t size){ - char *dirc = strdup(filename); - char *basec = strdup(filename); - char *dir = dirname(dirc); - char *base = basename(basec); - - std::string s(base); - std::stringstream ss; - std::string list[] = {"\\", " ", "\'", "\"", "*", "\?", "{", "}", ";", "<", ">", "|", "^", "&", "$", "#", "!", "`", "~"}; - std::string::size_type position; - for(auto substr : list){ - position = 0; - while((position = s.find_first_of(substr, position)) != std::string::npos){ - s.insert(position, "\\"); - position += 2; - } - } - if((position = s.find("global")) != std::string::npos){ - ss << "mv " << dir << "/" << s << " " << dir << "/" << s << "." << ++global_idx; - } - else if((position = s.find("debug")) != std::string::npos){ - ss << "mv " << dir << "/" << s << " " << dir << "/" << s << "." << ++debug_idx; - } - else if((position = s.find("warning")) != std::string::npos){ - ss << "mv " << dir << "/" << s << " " << dir << "/" << s << "." << ++warning_idx; - } - else if((position = s.find("trace")) != std::string::npos){ - ss << "mv " << dir << "/" << s << " " << dir << "/" << s << "." << ++trace_idx; - } - else if((position = s.find("error")) != std::string::npos){ - ss << "mv " << dir << "/" << s << " " << dir << "/" << s << "." << ++error_idx; - } - else if((position = s.find("fatal")) != std::string::npos){ - ss << "mv " << dir << "/" << s << " " << dir << "/" << s << "." << ++fatal_idx; - } - else{ - ss << "mv " << dir << "/" << s << " " << dir << "/" << s << "." << ++global_idx; - } -// std::cout << ss.str() << std::endl; - system(ss.str().c_str()); -} } // server } // milvus diff --git a/cpp/src/utils/LogUtil.h b/cpp/src/utils/LogUtil.h index 0d98e8133a..d6c9eff252 100644 --- a/cpp/src/utils/LogUtil.h +++ b/cpp/src/utils/LogUtil.h @@ -11,18 +11,7 @@ namespace zilliz { namespace milvus { namespace server { - -static unsigned global_idx = 0; -static unsigned debug_idx = 0; -static unsigned warning_idx = 0; -static unsigned trace_idx = 0; -static unsigned error_idx = 0; -static unsigned fatal_idx = 0; - int32_t InitLog(const std::string& log_config_file); - -void rolloutHandler(const char* filename, std::size_t size); - inline std::string GetFileName(std::string filename) { int pos = filename.find_last_of('/'); return filename.substr(pos + 1); From 8f2d7afd87f105f0ad6e32cd563d9d39fe5d5654 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Mon, 9 Sep 2019 20:27:58 +0800 Subject: [PATCH 3/3] recover the template file Former-commit-id: 8dd1549bc628c7567ca33c659bb3508a5877528a --- cpp/conf/log_config.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/conf/log_config.template b/cpp/conf/log_config.template index e4b0841d0b..cc33899589 100644 --- a/cpp/conf/log_config.template +++ b/cpp/conf/log_config.template @@ -6,7 +6,7 @@ TO_STANDARD_OUTPUT = false SUBSECOND_PRECISION = 3 PERFORMANCE_TRACKING = false - MAX_LOG_FILE_SIZE = 10240 ##209715200 ## Throw log files away after 200MB + MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB * DEBUG: FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-debug.log" ENABLED = true