admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove useless LogMgr (#16080)

Browse Source 
Signed-off-by: yudong.cai <yudong.cai@zilliz.com>
This commit is contained in:
Cai Yudong 2022-03-16 21:19:21 +08:00 committed by GitHub
parent d4ad785b76
commit d65cf5309f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 2 additions and 362 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/core/src/log/CMakeLists.txt
View File

@ -12,8 +12,6 @@
#-------------------------------------------------------------------------------
set(LOG_FILES ${MILVUS_ENGINE_SRC}/log/Log.cpp
${MILVUS_ENGINE_SRC}/log/Log.h
${MILVUS_ENGINE_SRC}/log/LogMgr.cpp
${MILVUS_ENGINE_SRC}/log/LogMgr.h
${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc
${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.h
)

258
internal/core/src/log/LogMgr.cpp
View File

@ -1,258 +0,0 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <libgen.h>
#include <cctype>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include <boost/filesystem.hpp>
#include "log/LogMgr.h"
#include "utils/Status.h"
#include "value/config/ServerConfig.h"
namespace milvus {
int LogMgr::trace_idx = 0;
int LogMgr::global_idx = 0;
int LogMgr::debug_idx = 0;
int LogMgr::info_idx = 0;
int LogMgr::warning_idx = 0;
int LogMgr::error_idx = 0;
int LogMgr::fatal_idx = 0;
int64_t LogMgr::logs_delete_exceeds = 1;
bool LogMgr::enable_log_delete = false;
Status
LogMgr::InitLog(bool trace_enable,
const std::string& level,
const std::string& logs_path,
const std::string& filename,
int64_t max_log_file_size,
int64_t log_rotate_num,
bool log_to_stdout,
bool log_to_file) {
try {
auto enables = parse_level(level);
enables["trace"] = trace_enable;
LogMgr log_mgr(logs_path);
log_mgr.Default()
.Filename(filename)
.Level(enables)
.To(log_to_stdout, log_to_file)
.Rotate(max_log_file_size, log_rotate_num)
.Setup();
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}
return Status::OK();
}
// TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename
void
LogMgr::RolloutHandler(const char* filename, std::size_t size, el::Level level) {
char* dirc = strdup(filename);
char* basec = strdup(filename);
char* dir = dirname(dirc);
char* base = basename(basec);
std::string s(base);
std::vector<std::string> list = {"\\", " ", "\'", "\"", "*", "\?", "{", "}", ";", "<",
">", "|", "^", "&", "$", "#", "!", "`", "~"};
std::string::size_type position;
for (auto& substr : list) {
position = 0;
while ((position = s.find_first_of(substr, position)) != std::string::npos) {
s.insert(position, "\\");
position += 2;
}
}
std::string m(std::string(dir) + "/" + s);
try {
switch (level) {
case el::Level::Trace: {
rename_and_delete(m, ++trace_idx);
break;
}
case el::Level::Global: {
rename_and_delete(m, ++global_idx);
break;
}
case el::Level::Debug: {
rename_and_delete(m, ++debug_idx);
break;
}
case el::Level::Info: {
rename_and_delete(m, ++info_idx);
break;
}
case el::Level::Warning: {
rename_and_delete(m, ++warning_idx);
break;
}
case el::Level::Error: {
rename_and_delete(m, ++error_idx);
break;
}
case el::Level::Fatal: {
rename_and_delete(m, ++fatal_idx);
break;
}
default: {
break;
}
}
} catch (const std::exception& exc) {
std::cerr << exc.what() << ". Exception throws from RolloutHandler." << std::endl;
}
}
LogMgr::LogMgr(std::string log_path) : logs_path_(std::move(log_path)) {
}
LogMgr&
LogMgr::Default() {
el_config_.setToDefault();
el_config_.setGlobally(el::ConfigurationType::Format, "[%datetime][%level]%msg");
el_config_.setGlobally(el::ConfigurationType::SubsecondPrecision, "3");
el_config_.setGlobally(el::ConfigurationType::PerformanceTracking, "false");
return *this;
}
LogMgr&
LogMgr::Filename(const std::string& filename) {
std::string logs_reg_path = logs_path_.rfind('/') == logs_path_.length() - 1 ? logs_path_ : logs_path_ + "/";
std::string log_file = logs_reg_path + filename;
/* Set set log file at Global level to make all level log output to the same log file*/
el_config_.set(el::Level::Global, el::ConfigurationType::Filename, log_file.c_str());
return *this;
}
LogMgr&
LogMgr::Level(std::unordered_map<std::string, bool>& enables) {
enable(el_config_, el::Level::Trace, enables["trace"]);
enable(el_config_, el::Level::Info, enables["info"]);
enable(el_config_, el::Level::Debug, enables["debug"]);
enable(el_config_, el::Level::Warning, enables["warning"]);
enable(el_config_, el::Level::Error, enables["error"]);
enable(el_config_, el::Level::Fatal, enables["fatal"]);
return *this;
}
LogMgr&
LogMgr::To(bool log_to_stdout, bool log_to_file) {
el_config_.setGlobally(el::ConfigurationType::ToStandardOutput, (log_to_stdout ? "true" : "false"));
el_config_.setGlobally(el::ConfigurationType::ToFile, (log_to_file ? "true" : "false"));
return *this;
}
LogMgr&
LogMgr::Rotate(int64_t max_log_file_size, int64_t log_rotate_num) {
if (max_log_file_size < MAX_LOG_FILE_SIZE_MIN || max_log_file_size > MAX_LOG_FILE_SIZE_MAX) {
std::string msg = "max_log_file_size must in range[" + std::to_string(MAX_LOG_FILE_SIZE_MIN) + ", " +
std::to_string(MAX_LOG_FILE_SIZE_MAX) + "], now is " + std::to_string(max_log_file_size);
throw std::runtime_error(msg);
}
el_config_.setGlobally(el::ConfigurationType::MaxLogFileSize, std::to_string(max_log_file_size));
el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck);
el::Helpers::installPreRollOutCallback(LogMgr::RolloutHandler);
el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog);
// set delete_exceeds = 0 means disable throw away log file even they reach certain limit.
if (log_rotate_num != 0) {
if (log_rotate_num < LOG_ROTATE_NUM_MIN || log_rotate_num > LOG_ROTATE_NUM_MAX) {
std::string msg = "log_rotate_num must in range[" + std::to_string(LOG_ROTATE_NUM_MIN) + ", " +
std::to_string(LOG_ROTATE_NUM_MAX) + "], now is " + std::to_string(log_rotate_num);
throw std::runtime_error(msg);
}
/* global variable */
enable_log_delete = true;
logs_delete_exceeds = log_rotate_num;
}
return *this;
}
void
LogMgr::Setup() {
el::Loggers::reconfigureLogger("default", el_config_);
}
void
LogMgr::rename_and_delete(const std::string& filename, int64_t idx) {
std::string target_filename = filename + "." + std::to_string(idx);
rename(filename.c_str(), target_filename.c_str());
if (enable_log_delete && idx - logs_delete_exceeds > 0) {
std::string to_delete = filename + "." + std::to_string(trace_idx - logs_delete_exceeds);
boost::filesystem::remove(to_delete);
}
}
std::unordered_map<std::string, bool>
LogMgr::parse_level(const std::string& level) {
std::unordered_map<std::string, bool> enables{
{"debug", false}, {"info", false}, {"warning", false}, {"error", false}, {"fatal", false},
};
std::unordered_map<std::string, int64_t> level_to_int{
{"debug", 5}, {"info", 4}, {"warning", 3}, {"error", 2}, {"fatal", 1},
};
switch (level_to_int[level]) {
case 5:
enables["debug"] = true;
case 4:
enables["info"] = true;
case 3:
enables["warning"] = true;
case 2:
enables["error"] = true;
case 1:
enables["fatal"] = true;
break;
default: {
std::string msg = "Cannot parse level " + level +
": invalid log level, must be one of debug, info, warning, error, fatal.";
throw std::runtime_error(msg);
}
}
return enables;
}
void
LogMgr::enable(el::Configurations& default_conf, el::Level level, bool enable) {
std::string enable_str = enable ? "true" : "false";
default_conf.set(level, el::ConfigurationType::Enabled, enable_str);
}
} // namespace milvus

100
internal/core/src/log/LogMgr.h
View File

@ -1,100 +0,0 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "easyloggingpp/easylogging++.h"
#include "utils/Status.h"
#include "value/config/ServerConfig.h"
#include <sstream>
#include <string>
#include <unordered_map>
namespace milvus {
class LogMgr {
public:
static Status
InitLog(bool trace_enable,
const std::string& level,
const std::string& logs_path,
const std::string& filename,
int64_t max_log_file_size,
int64_t delete_exceeds,
bool log_to_stdout,
bool log_to_file);
static void
RolloutHandler(const char* filename, std::size_t size, el::Level level);
private:
explicit LogMgr(std::string log_path);
LogMgr&
Default();
LogMgr&
Filename(const std::string& filename);
/* Non-const for fiu to injecting error */
LogMgr&
Level(std::unordered_map<std::string, bool>& enables);
LogMgr&
To(bool log_to_stdout, bool log_to_file);
LogMgr&
Rotate(int64_t max_log_file_size, int64_t log_rotate_num);
void
Setup();
private:
static void
rename_and_delete(const std::string& filename, int64_t idx);
static std::unordered_map<std::string, bool>
parse_level(const std::string& level);
/**
* @brief Configures if output corresponding level log
*/
static void
enable(el::Configurations& default_conf, el::Level level, bool enable);
private:
el::Configurations el_config_;
std::string logs_path_;
private:
static int trace_idx;
static int global_idx;
static int debug_idx;
static int info_idx;
static int warning_idx;
static int error_idx;
static int fatal_idx;
static int64_t logs_delete_exceeds;
static bool enable_log_delete;
const int64_t MAX_LOG_FILE_SIZE_MIN = 536870912; /* 512 MB */
const int64_t MAX_LOG_FILE_SIZE_MAX = 4294967296; /* 4 GB */
const int64_t LOG_ROTATE_NUM_MIN = 0;
const int64_t LOG_ROTATE_NUM_MAX = 1024;
};
} // namespace milvus

4
internal/core/thirdparty/knowhere/CMakeLists.txt vendored
View File

@ -27,8 +27,8 @@ macro(build_knowhere)
set(KNOWHERE_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
-DCMAKE_BUILD_TYPE=Release
"-DCMAKE_INSTALL_PREFIX=${KNOWHERE_INSTALL_PREFIX}"
-DCMAKE_BUILD_TYPE={BUILD_TYPE}
-DCMAKE_INSTALL_PREFIX=${KNOWHERE_INSTALL_PREFIX}
)
externalproject_add(knowhere_ep