diff --git a/configs/easylogging.yaml b/configs/easylogging.yaml new file mode 100644 index 0000000000..9a2d5ad2c4 --- /dev/null +++ b/configs/easylogging.yaml @@ -0,0 +1,25 @@ +* GLOBAL: + FORMAT = "%datetime | %level | %logger | %msg" + FILENAME = "/var/lib/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-easylog.log" + ENABLED = false + TO_FILE = false + TO_STANDARD_OUTPUT = false + SUBSECOND_PRECISION = 3 + PERFORMANCE_TRACKING = false + MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB +* INFO: + ENABLED = false +* TRACE: + ENABLED = false +* DEBUG: + ENABLED = false +* WARNING: + ENABLED = false +* ERROR: + ENABLED = true +* FATAL: + ENABLED = true +* VERBOSE: + FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" + TO_FILE = false + TO_STANDARD_OUTPUT = false diff --git a/internal/core/src/config/ConfigKnowhere.cpp b/internal/core/src/config/ConfigKnowhere.cpp index c00fd710a8..5cc9fe7726 100644 --- a/internal/core/src/config/ConfigKnowhere.cpp +++ b/internal/core/src/config/ConfigKnowhere.cpp @@ -27,23 +27,24 @@ namespace milvus::config { std::once_flag init_knowhere_once_; void -KnowhereInitImpl() { - auto init = []() { +KnowhereInitImpl(const char* conf_file) { + auto init = [&]() { knowhere::KnowhereConfig::SetBlasThreshold(16384); knowhere::KnowhereConfig::SetEarlyStopThreshold(0); knowhere::KnowhereConfig::SetLogHandler(); knowhere::KnowhereConfig::SetStatisticsLevel(0); + +#ifdef EMBEDDED_MILVUS + // always disable all logs for embedded milvus. el::Configurations el_conf; - el_conf.setGlobally(el::ConfigurationType::Enabled, std::to_string(false)); -#if defined(EMBEDDED_MILVUS) - // Disable all logs for embedded milvus. - el_conf.set(el::Level::Trace, el::ConfigurationType::Enabled, "false"); - el_conf.set(el::Level::Debug, el::ConfigurationType::Enabled, "false"); - el_conf.set(el::Level::Info, el::ConfigurationType::Enabled, "false"); - el_conf.set(el::Level::Warning, el::ConfigurationType::Enabled, "false"); - el_conf.set(el::Level::Error, el::ConfigurationType::Enabled, "false"); - el_conf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "false"); - el::Loggers::reconfigureLogger("default", el_conf); + el_conf.setGlobally(el::ConfigurationType::Enabled, "false"); + el::Loggers::reconfigureAllLoggers(el_conf); +#else + if (conf_file != nullptr) { + el::Configurations el_conf(conf_file); + el::Loggers::reconfigureAllLoggers(el_conf); + LOG_SERVER_DEBUG_ << "Config easylogging with yaml file: " << conf_file; + } #endif }; diff --git a/internal/core/src/config/ConfigKnowhere.h b/internal/core/src/config/ConfigKnowhere.h index 290fb16335..0b806bdbbe 100644 --- a/internal/core/src/config/ConfigKnowhere.h +++ b/internal/core/src/config/ConfigKnowhere.h @@ -20,7 +20,7 @@ namespace milvus::config { void -KnowhereInitImpl(); +KnowhereInitImpl(const char*); std::string KnowhereSetSimdType(const char*); diff --git a/internal/core/src/indexbuilder/init_c.cpp b/internal/core/src/indexbuilder/init_c.cpp index e88c9bbee6..40b9912ce3 100644 --- a/internal/core/src/indexbuilder/init_c.cpp +++ b/internal/core/src/indexbuilder/init_c.cpp @@ -14,8 +14,8 @@ #include "indexbuilder/init_c.h" void -IndexBuilderInit() { - milvus::config::KnowhereInitImpl(); +IndexBuilderInit(const char* conf_file) { + milvus::config::KnowhereInitImpl(conf_file); } // return value must be freed by the caller diff --git a/internal/core/src/indexbuilder/init_c.h b/internal/core/src/indexbuilder/init_c.h index dbaa0c1cf1..223090ca93 100644 --- a/internal/core/src/indexbuilder/init_c.h +++ b/internal/core/src/indexbuilder/init_c.h @@ -16,7 +16,7 @@ extern "C" { #endif void -IndexBuilderInit(); +IndexBuilderInit(const char*); // return value must be freed by the caller char* diff --git a/internal/core/src/segcore/segcore_init_c.cpp b/internal/core/src/segcore/segcore_init_c.cpp index 5b2bbd600f..d5abb661e2 100644 --- a/internal/core/src/segcore/segcore_init_c.cpp +++ b/internal/core/src/segcore/segcore_init_c.cpp @@ -16,20 +16,8 @@ namespace milvus::segcore { extern "C" void -SegcoreInit() { - milvus::config::KnowhereInitImpl(); -#if defined(EMBEDDED_MILVUS) - el::Configurations defaultConf; - defaultConf.setToDefault(); - // Disable all logs for embedded milvus. - defaultConf.set(el::Level::Trace, el::ConfigurationType::Enabled, "false"); - defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, "false"); - defaultConf.set(el::Level::Info, el::ConfigurationType::Enabled, "false"); - defaultConf.set(el::Level::Warning, el::ConfigurationType::Enabled, "false"); - defaultConf.set(el::Level::Error, el::ConfigurationType::Enabled, "false"); - defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "false"); - el::Loggers::reconfigureLogger("default", defaultConf); -#endif +SegcoreInit(const char* conf_file) { + milvus::config::KnowhereInitImpl(conf_file); } extern "C" void diff --git a/internal/core/src/segcore/segcore_init_c.h b/internal/core/src/segcore/segcore_init_c.h index 9a97552ade..5f9803d242 100644 --- a/internal/core/src/segcore/segcore_init_c.h +++ b/internal/core/src/segcore/segcore_init_c.h @@ -16,7 +16,7 @@ extern "C" { #endif void -SegcoreInit(); +SegcoreInit(const char*); void SegcoreSetChunkRows(const int64_t); diff --git a/internal/core/unittest/test_init.cpp b/internal/core/unittest/test_init.cpp index e698185113..d15dfefc23 100644 --- a/internal/core/unittest/test_init.cpp +++ b/internal/core/unittest/test_init.cpp @@ -17,7 +17,7 @@ TEST(Init, Naive) { using namespace milvus; using namespace milvus::segcore; - SegcoreInit(); + SegcoreInit(nullptr); SegcoreSetChunkRows(32768); SegcoreSetSimdType("auto"); } diff --git a/internal/indexnode/indexnode.go b/internal/indexnode/indexnode.go index 15caa61afe..0c3cf7ce3c 100644 --- a/internal/indexnode/indexnode.go +++ b/internal/indexnode/indexnode.go @@ -36,6 +36,7 @@ import ( "io" "math/rand" "os" + "path" "strconv" "sync" "sync/atomic" @@ -146,7 +147,9 @@ func (i *IndexNode) Register() error { } func (i *IndexNode) initKnowhere() { - C.IndexBuilderInit() + cEasyloggingYaml := C.CString(path.Join(Params.BaseTable.GetConfigDir(), paramtable.DefaultEasyloggingYaml)) + C.IndexBuilderInit(cEasyloggingYaml) + C.free(unsafe.Pointer(cEasyloggingYaml)) // override index builder SIMD type cSimdType := C.CString(Params.CommonCfg.SimdType) diff --git a/internal/querynode/query_node.go b/internal/querynode/query_node.go index 6669fbe2c5..1bdc0a40e6 100644 --- a/internal/querynode/query_node.go +++ b/internal/querynode/query_node.go @@ -36,6 +36,7 @@ import ( "errors" "fmt" "os" + "path" "path/filepath" "strconv" "sync" @@ -176,7 +177,9 @@ func (node *QueryNode) Register() error { // InitSegcore set init params of segCore, such as chunckRows, SIMD type... func (node *QueryNode) InitSegcore() { - C.SegcoreInit() + cEasyloggingYaml := C.CString(path.Join(Params.BaseTable.GetConfigDir(), paramtable.DefaultEasyloggingYaml)) + C.SegcoreInit(cEasyloggingYaml) + C.free(unsafe.Pointer(cEasyloggingYaml)) // override segcore chunk size cChunkRows := C.int64_t(Params.QueryNodeCfg.ChunkRows) diff --git a/internal/util/paramtable/base_table.go b/internal/util/paramtable/base_table.go index 761ab66399..b4fd8f5a3f 100644 --- a/internal/util/paramtable/base_table.go +++ b/internal/util/paramtable/base_table.go @@ -35,6 +35,8 @@ import ( type UniqueID = typeutil.UniqueID const ( + DefaultMilvusYaml = "milvus.yaml" + DefaultEasyloggingYaml = "easylogging.yaml" DefaultMinioHost = "localhost" DefaultMinioPort = "9000" DefaultMinioAccessKey = "minioadmin" @@ -49,7 +51,7 @@ const ( DefaultEnvPrefix = "milvus" ) -var DefaultYaml = "milvus.yaml" +var defaultYaml = DefaultMilvusYaml // Base abstracts BaseTable // TODO: it's never used, consider to substitute BaseTable or to remove it @@ -79,7 +81,7 @@ type BaseTable struct { // GlobalInitWithYaml should be called only in standalone and embedded Milvus. func (gp *BaseTable) GlobalInitWithYaml(yaml string) { gp.once.Do(func() { - DefaultYaml = yaml + defaultYaml = yaml gp.Init() }) } @@ -88,7 +90,7 @@ func (gp *BaseTable) GlobalInitWithYaml(yaml string) { func (gp *BaseTable) Init() { gp.params = memkv.NewMemoryKV() gp.configDir = gp.initConfPath() - gp.loadFromYaml(DefaultYaml) + gp.loadFromYaml(defaultYaml) gp.tryLoadFromEnv() gp.InitLogCfg() } diff --git a/internal/util/paramtable/base_table_test.go b/internal/util/paramtable/base_table_test.go index b7b7241502..1c0f996a38 100644 --- a/internal/util/paramtable/base_table_test.go +++ b/internal/util/paramtable/base_table_test.go @@ -22,8 +22,6 @@ import ( "github.com/stretchr/testify/assert" ) -const defaultYaml = "milvus.yaml" - var baseParams = BaseTable{} func TestMain(m *testing.M) {