admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-05 10:22:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.5.0-improve_log' into 'branch-0.5.0'

Browse Source 
MS-552 Add and change the easylogging library

See merge request megasearch/milvus!567

Former-commit-id: 654becfce1696734267d5993e4773a5c914ee8b9
This commit is contained in:
jinhai 2019-09-17 09:54:41 +08:00
commit e25c0aa995
27 changed files with 7728 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/CHANGELOG.md
View File

@ -7,6 +7,7 @@ Please mark all change in change log and use the ticket from JIRA.
## Bug
## Improvement
- MS-552 - Add and change the easylogging library
## New Feature

88
cpp/cmake/ThirdPartyPackages.cmake
View File

@ -19,7 +19,6 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES
ARROW
BOOST
BZip2
Easylogging++
FAISS
GTest
Knowhere
@ -56,8 +55,6 @@ macro(build_dependency DEPENDENCY_NAME)
build_arrow()
elseif("${DEPENDENCY_NAME}" STREQUAL "BZip2")
build_bzip2()
elseif("${DEPENDENCY_NAME}" STREQUAL "Easylogging++")
build_easyloggingpp()
elseif("${DEPENDENCY_NAME}" STREQUAL "FAISS")
build_faiss()
elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest")
@ -295,13 +292,6 @@ else()
endif()
set(BZIP2_MD5 "00b516f4704d4a7cb50a1d97e6e8e15b")
if(DEFINED ENV{MILVUS_EASYLOGGINGPP_URL})
set(EASYLOGGINGPP_SOURCE_URL "$ENV{MILVUS_EASYLOGGINGPP_URL}")
else()
set(EASYLOGGINGPP_SOURCE_URL "https://github.com/zuhd-org/easyloggingpp/archive/${EASYLOGGINGPP_VERSION}.tar.gz")
endif()
set(EASYLOGGINGPP_MD5 "b78cd319db4be9b639927657b8aa7732")
if(DEFINED ENV{MILVUS_FAISS_URL})
set(FAISS_SOURCE_URL "$ENV{MILVUS_FAISS_URL}")
else()
@ -814,84 +804,6 @@ if(MILVUS_WITH_KNOWHERE)
include_directories(SYSTEM "${KNOWHERE_INCLUDE_DIR}/SPTAG/AnnService")
endif()
# ----------------------------------------------------------------------
# Easylogging++
macro(build_easyloggingpp)
message(STATUS "Building Easylogging++-${EASYLOGGINGPP_VERSION} from source")
set(EASYLOGGINGPP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/easyloggingpp_ep-prefix/src/easyloggingpp_ep")
set(EASYLOGGINGPP_INCLUDE_DIR "${EASYLOGGINGPP_PREFIX}/include")
set(EASYLOGGINGPP_STATIC_LIB
"${EASYLOGGINGPP_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}easyloggingpp${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(EASYLOGGINGPP_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
"-DCMAKE_INSTALL_PREFIX=${EASYLOGGINGPP_PREFIX}"
-DCMAKE_INSTALL_LIBDIR=lib
-Dtest=OFF
-Dbuild_static_lib=ON)
if(USE_JFROG_CACHE STREQUAL "ON")
set(EASYLOGGINGPP_CACHE_PACKAGE_NAME "easyloggingpp_${EASYLOGGINGPP_MD5}.tar.gz")
set(EASYLOGGINGPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}")
set(EASYLOGGINGPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${EASYLOGGINGPP_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote cache file ${EASYLOGGINGPP_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(easyloggingpp_ep
URL
${EASYLOGGINGPP_SOURCE_URL}
${EP_LOG_OPTIONS}
CMAKE_ARGS
${EASYLOGGINGPP_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_BYPRODUCTS
${EASYLOGGINGPP_STATIC_LIB})
ExternalProject_Create_Cache(easyloggingpp_ep ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/easyloggingpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${EASYLOGGINGPP_CACHE_URL})
else()
file(DOWNLOAD ${EASYLOGGINGPP_CACHE_URL} ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${EASYLOGGINGPP_CACHE_URL} TO ${EASYLOGGINGPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(easyloggingpp_ep ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(easyloggingpp_ep
URL
${EASYLOGGINGPP_SOURCE_URL}
${EP_LOG_OPTIONS}
CMAKE_ARGS
${EASYLOGGINGPP_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_BYPRODUCTS
${EASYLOGGINGPP_STATIC_LIB})
endif()
file(MAKE_DIRECTORY "${EASYLOGGINGPP_INCLUDE_DIR}")
add_library(easyloggingpp STATIC IMPORTED)
set_target_properties(
easyloggingpp
PROPERTIES IMPORTED_LOCATION "${EASYLOGGINGPP_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${EASYLOGGINGPP_INCLUDE_DIR}")
add_dependencies(easyloggingpp easyloggingpp_ep)
endmacro()
if(MILVUS_WITH_EASYLOGGINGPP)
resolve_dependency(Easylogging++)
get_target_property(EASYLOGGINGPP_INCLUDE_DIR easyloggingpp INTERFACE_INCLUDE_DIRECTORIES)
link_directories(SYSTEM "${EASYLOGGINGPP_PREFIX}/lib")
include_directories(SYSTEM "${EASYLOGGINGPP_INCLUDE_DIR}")
endif()
# ----------------------------------------------------------------------
# OpenBLAS

3
cpp/src/CMakeLists.txt
View File

@ -73,6 +73,7 @@ set(db_files
${db_scheduler_files}
${metrics_files}
${knowhere_files}
${utils_files}
)
set(s3_client_files
@ -94,7 +95,6 @@ set(client_grpc_lib
set(third_party_libs
knowhere
easyloggingpp
sqlite
${client_grpc_lib}
yaml-cpp
@ -145,7 +145,6 @@ target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs})
add_library(metrics STATIC ${metrics_files})
set(metrics_lib
easyloggingpp
yaml-cpp
prometheus-cpp-push
prometheus-cpp-pull

2
cpp/src/db/Log.h
View File

@ -5,7 +5,7 @@
******************************************************************************/
#pragma once
#include <easylogging++.h>
#include "utils/easylogging++.h"
namespace zilliz {
namespace milvus {

2
cpp/src/db/Options.cpp
View File

@ -5,7 +5,7 @@
******************************************************************************/
#include <stdlib.h>
#include <assert.h>
#include <easylogging++.h>
#include "utils/easylogging++.h"
#include <boost/algorithm/string.hpp>
#include "Options.h"

2
cpp/src/main.cpp
View File

@ -11,7 +11,7 @@
#include <cstring>
#include <string>
#include <signal.h>
#include <easylogging++.h>
#include "utils/easylogging++.h"
#include "metrics/Metrics.h"
#include "utils/SignalUtil.h"

2
cpp/src/utils/Log.h
View File

@ -6,7 +6,7 @@
#pragma once
#include "Error.h"
#include <easylogging++.h>
#include "easylogging++.h"
namespace zilliz {
namespace milvus {

17
cpp/src/utils/LogUtil.cpp
View File

@ -5,8 +5,8 @@
////////////////////////////////////////////////////////////////////////////////
#include "LogUtil.h"
#include "server/ServerConfig.h"
#include "easylogging++.h"
#include <easylogging++.h>
#include <ctype.h>
#include <string>
@ -27,7 +27,7 @@ static int fatal_idx = 0;
}
// TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename
void RolloutHandler(const char *filename, std::size_t size) {
void RolloutHandler(const char *filename, std::size_t size, el::Level level) {
char *dirc = strdup(filename);
char *basec = strdup(filename);
char *dir = dirname(dirc);
@ -48,22 +48,22 @@ void RolloutHandler(const char *filename, std::size_t size) {
int ret;
std::string m(std::string(dir) + "/" + s);
s = m;
if ((position = s.find("global")) != std::string::npos) {
if (level == el::Level::Global) {
s.append("." + std::to_string(++global_idx));
ret = rename(m.c_str(), s.c_str());
} else if ((position = s.find("debug")) != std::string::npos) {
} else if (level == el::Level::Debug) {
s.append("." + std::to_string(++debug_idx));
ret = rename(m.c_str(), s.c_str());
} else if ((position = s.find("warning")) != std::string::npos) {
} else if (level == el::Level::Warning) {
s.append("." + std::to_string(++warning_idx));
ret = rename(m.c_str(), s.c_str());
} else if ((position = s.find("trace")) != std::string::npos) {
} else if (level == el::Level::Trace) {
s.append("." + std::to_string(++trace_idx));
ret = rename(m.c_str(), s.c_str());
} else if ((position = s.find("error")) != std::string::npos) {
} else if (level == el::Level::Error) {
s.append("." + std::to_string(++error_idx));
ret = rename(m.c_str(), s.c_str());
} else if ((position = s.find("fatal")) != std::string::npos) {
} else if (level == el::Level::Fatal) {
s.append("." + std::to_string(++fatal_idx));
ret = rename(m.c_str(), s.c_str());
} else {
@ -113,6 +113,7 @@ int32_t InitLog(const std::string &log_config_file) {
el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck);
el::Helpers::installPreRollOutCallback(RolloutHandler);
el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog);
return 0;
}

3
cpp/src/utils/LogUtil.h
View File

@ -7,6 +7,7 @@
#include <string>
#include <sstream>
#include "easylogging++.h"
namespace zilliz {
namespace milvus {
@ -16,7 +17,7 @@ inline std::string GetFileName(std::string filename) {
int pos = filename.find_last_of('/');
return filename.substr(pos + 1);
}
void RolloutHandler(const char *filename, std::size_t size);
void RolloutHandler(const char *filename, std::size_t size, el::Level level);
#define SHOW_LOCATION
#ifdef SHOW_LOCATION

3112
cpp/src/utils/easylogging++.cc Normal file
View File

File diff suppressed because it is too large Load Diff

4569
cpp/src/utils/easylogging++.h Normal file
View File

File diff suppressed because it is too large Load Diff

2
cpp/src/wrapper/knowhere/wrapper_log.h
View File

@ -6,7 +6,7 @@
#pragma once
#include <easylogging++.h>
#include "utils/easylogging++.h"
namespace zilliz {
namespace milvus {

1
cpp/unittest/CMakeLists.txt
View File

@ -25,7 +25,6 @@ set(unittest_libs
gmock
gtest_main
gmock_main
easyloggingpp
pthread
metrics
gfortran

4
cpp/unittest/db/CMakeLists.txt
View File

@ -19,7 +19,9 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs)
aux_source_directory(./ test_srcs)
set(util_files
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc
${MILVUS_ENGINE_SRC}/utils/easylogging++.h)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)

2
cpp/unittest/db/db_tests.cpp
View File

@ -12,7 +12,7 @@
#include "utils/CommonUtil.h"
#include <gtest/gtest.h>
#include <easylogging++.h>
#include "utils/easylogging++.h"
#include <boost/filesystem.hpp>

2
cpp/unittest/db/meta_tests.cpp
View File

@ -5,7 +5,7 @@
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include <thread>
#include <easylogging++.h>
#include "utils/easylogging++.h"
#include <stdlib.h>
#include <time.h>

2
cpp/unittest/db/misc_test.cpp
View File

@ -5,7 +5,7 @@
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include <thread>
#include <easylogging++.h>
#include "utils/easylogging++.h"
#include <boost/filesystem.hpp>
#include "db/Exception.h"

2
cpp/unittest/db/mysql_db_test.cpp
View File

@ -9,7 +9,7 @@
#include "db/meta/MetaConsts.h"
#include <gtest/gtest.h>
#include <easylogging++.h>
#include "utils/easylogging++.h"
#include <boost/filesystem.hpp>
#include <thread>

2
cpp/unittest/db/mysql_meta_test.cpp
View File

@ -5,7 +5,7 @@
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include <thread>
#include <easylogging++.h>
#include "utils/easylogging++.h"
#include <stdlib.h>
#include <time.h>

2
cpp/unittest/db/scheduler_test.cpp
View File

@ -5,7 +5,7 @@
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include <thread>
#include <easylogging++.h>
#include "utils/easylogging++.h"
#include <boost/filesystem.hpp>
#include "db/scheduler/TaskScheduler.h"

4
cpp/unittest/knowhere/CMakeLists.txt
View File

@ -7,7 +7,9 @@ set(knowhere_src
${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_index.cpp)
set(helper
utils.cpp)
utils.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc
${MILVUS_ENGINE_SRC}/utils/easylogging++.h)
set(knowhere_libs
knowhere

2
cpp/unittest/knowhere/knowhere_test.cpp
View File

@ -5,7 +5,7 @@
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include <easylogging++.h>
#include "utils/easylogging++.h"
#include <wrapper/knowhere/vec_index.h>
#include "knowhere/index/vector_index/gpu_ivf.h"

2
cpp/unittest/main.cpp
View File

@ -5,7 +5,7 @@
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include <easylogging++.h>
#include "utils/easylogging++.h"
#include "server/ServerConfig.h"
#include "utils/CommonUtil.h"

4
cpp/unittest/metrics/CMakeLists.txt
View File

@ -18,7 +18,9 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/src/metrics metrics_src)
aux_source_directory(./ test_srcs)
set(util_files
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc
${MILVUS_ENGINE_SRC}/utils/easylogging++.h)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)

5
cpp/unittest/scheduler/CMakeLists.txt
View File

@ -21,7 +21,10 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs)
aux_source_directory(./ test_srcs)
set(util_files ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
set(util_files
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc
${MILVUS_ENGINE_SRC}/utils/easylogging++.h)
set(db_scheduler_srcs
${scheduler_files}

5
cpp/unittest/server/CMakeLists.txt
View File

@ -39,6 +39,11 @@ set(db_scheduler_srcs
${scheduler_task_files}
)
set(util_files
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc
${MILVUS_ENGINE_SRC}/utils/easylogging++.h)
set(db_src
${config_files}
${cache_srcs}

5
cpp/unittest/server/util_test.cpp
View File

@ -5,7 +5,7 @@
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include <thread>
#include <easylogging++.h>
#include "utils/easylogging++.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <boost/filesystem.hpp>
@ -290,6 +290,7 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST){
std::string dir1 = "/tmp/milvus_test";
std::string dir2 = "/tmp/milvus_test/log_test";
std::string filename[6] = {"log_global.log", "log_debug.log", "log_warning.log", "log_trace.log", "log_error.log", "log_fatal.log"};
el::Level list[6] = {el::Level::Global, el::Level::Debug, el::Level::Warning, el::Level::Trace, el::Level::Error, el::Level::Fatal};
mkdir(dir1.c_str(), S_IRWXU);
mkdir(dir2.c_str(), S_IRWXU);
@ -300,7 +301,7 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST){
file.open(tmp.c_str());
file << "zilliz" << std::endl;
server::RolloutHandler(tmp.c_str(), 0);
server::RolloutHandler(tmp.c_str(), 0, list[i]);
tmp.append(".1");
std::ifstream file2;