Support easylogging config for segcore and knowhere (#16751)

Signed-off-by: yudong.cai <yudong.cai@zilliz.com>

configs/easylogging.yaml

@@ -0,0 +1,25 @@
+* GLOBAL:
+    FORMAT                  =   "%datetime | %level | %logger | %msg"
+    FILENAME                =   "/var/lib/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-easylog.log"
+    ENABLED                 =   false
+    TO_FILE                 =   false
+    TO_STANDARD_OUTPUT      =   false
+    SUBSECOND_PRECISION     =   3
+    PERFORMANCE_TRACKING    =   false
+    MAX_LOG_FILE_SIZE       =   209715200 ## Throw log files away after 200MB
+* INFO:
+    ENABLED                 =   false
+* TRACE:
+    ENABLED                 =   false
+* DEBUG:
+    ENABLED                 =   false
+* WARNING:
+    ENABLED                 =   false
+* ERROR:
+    ENABLED                 =   true
+* FATAL:
+    ENABLED                 =   true
+* VERBOSE:
+    FORMAT                  =   "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
+    TO_FILE                 =   false
+    TO_STANDARD_OUTPUT      =   false

internal/core/src/config/ConfigKnowhere.cpp

@@ -27,23 +27,24 @@ namespace milvus::config {
 std::once_flag init_knowhere_once_;
 
 void
-KnowhereInitImpl() {
+KnowhereInitImpl(const char* conf_file) {
-    auto init = []() {
+    auto init = [&]() {
         knowhere::KnowhereConfig::SetBlasThreshold(16384);
         knowhere::KnowhereConfig::SetEarlyStopThreshold(0);
         knowhere::KnowhereConfig::SetLogHandler();
         knowhere::KnowhereConfig::SetStatisticsLevel(0);
+
+#ifdef EMBEDDED_MILVUS
+        // always disable all logs for embedded milvus.
         el::Configurations el_conf;
-        el_conf.setGlobally(el::ConfigurationType::Enabled, std::to_string(false));
+        el_conf.setGlobally(el::ConfigurationType::Enabled, "false");
-#if defined(EMBEDDED_MILVUS)
+        el::Loggers::reconfigureAllLoggers(el_conf);
-        // Disable all logs for embedded milvus.
+#else
-        el_conf.set(el::Level::Trace, el::ConfigurationType::Enabled, "false");
+        if (conf_file != nullptr) {
-        el_conf.set(el::Level::Debug, el::ConfigurationType::Enabled, "false");
+            el::Configurations el_conf(conf_file);
-        el_conf.set(el::Level::Info, el::ConfigurationType::Enabled, "false");
+            el::Loggers::reconfigureAllLoggers(el_conf);
-        el_conf.set(el::Level::Warning, el::ConfigurationType::Enabled, "false");
+            LOG_SERVER_DEBUG_ << "Config easylogging with yaml file: " << conf_file;
-        el_conf.set(el::Level::Error, el::ConfigurationType::Enabled, "false");
+        }
-        el_conf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "false");
-        el::Loggers::reconfigureLogger("default", el_conf);
 #endif
     };
 

internal/core/src/config/ConfigKnowhere.h

@@ -20,7 +20,7 @@
 namespace milvus::config {
 
 void
-KnowhereInitImpl();
+KnowhereInitImpl(const char*);
 
 std::string
 KnowhereSetSimdType(const char*);

internal/core/src/indexbuilder/init_c.cpp

@@ -14,8 +14,8 @@
 #include "indexbuilder/init_c.h"
 
 void
-IndexBuilderInit() {
+IndexBuilderInit(const char* conf_file) {
-    milvus::config::KnowhereInitImpl();
+    milvus::config::KnowhereInitImpl(conf_file);
 }
 
 // return value must be freed by the caller

internal/core/src/indexbuilder/init_c.h

@@ -16,7 +16,7 @@ extern "C" {
 #endif
 
 void
-IndexBuilderInit();
+IndexBuilderInit(const char*);
 
 // return value must be freed by the caller
 char*

internal/core/src/segcore/segcore_init_c.cpp

@@ -16,20 +16,8 @@
 
 namespace milvus::segcore {
 extern "C" void
-SegcoreInit() {
+SegcoreInit(const char* conf_file) {
-    milvus::config::KnowhereInitImpl();
+    milvus::config::KnowhereInitImpl(conf_file);
-#if defined(EMBEDDED_MILVUS)
-    el::Configurations defaultConf;
-    defaultConf.setToDefault();
-    // Disable all logs for embedded milvus.
-    defaultConf.set(el::Level::Trace, el::ConfigurationType::Enabled, "false");
-    defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, "false");
-    defaultConf.set(el::Level::Info, el::ConfigurationType::Enabled, "false");
-    defaultConf.set(el::Level::Warning, el::ConfigurationType::Enabled, "false");
-    defaultConf.set(el::Level::Error, el::ConfigurationType::Enabled, "false");
-    defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "false");
-    el::Loggers::reconfigureLogger("default", defaultConf);
-#endif
 }
 
 extern "C" void

internal/core/src/segcore/segcore_init_c.h

@@ -16,7 +16,7 @@ extern "C" {
 #endif
 
 void
-SegcoreInit();
+SegcoreInit(const char*);
 
 void
 SegcoreSetChunkRows(const int64_t);

internal/core/unittest/test_init.cpp

@@ -17,7 +17,7 @@
 TEST(Init, Naive) {
     using namespace milvus;
     using namespace milvus::segcore;
-    SegcoreInit();
+    SegcoreInit(nullptr);
     SegcoreSetChunkRows(32768);
     SegcoreSetSimdType("auto");
 }

internal/indexnode/indexnode.go

@@ -36,6 +36,7 @@ import (
 	"io"
 	"math/rand"
 	"os"
+	"path"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -146,7 +147,9 @@ func (i *IndexNode) Register() error {
 }
 
 func (i *IndexNode) initKnowhere() {
-	C.IndexBuilderInit()
+	cEasyloggingYaml := C.CString(path.Join(Params.BaseTable.GetConfigDir(), paramtable.DefaultEasyloggingYaml))
+	C.IndexBuilderInit(cEasyloggingYaml)
+	C.free(unsafe.Pointer(cEasyloggingYaml))
 
 	// override index builder SIMD type
 	cSimdType := C.CString(Params.CommonCfg.SimdType)

internal/querynode/query_node.go

@@ -36,6 +36,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"strconv"
 	"sync"
@@ -176,7 +177,9 @@ func (node *QueryNode) Register() error {
 
 // InitSegcore set init params of segCore, such as chunckRows, SIMD type...
 func (node *QueryNode) InitSegcore() {
-	C.SegcoreInit()
+	cEasyloggingYaml := C.CString(path.Join(Params.BaseTable.GetConfigDir(), paramtable.DefaultEasyloggingYaml))
+	C.SegcoreInit(cEasyloggingYaml)
+	C.free(unsafe.Pointer(cEasyloggingYaml))
 
 	// override segcore chunk size
 	cChunkRows := C.int64_t(Params.QueryNodeCfg.ChunkRows)

internal/util/paramtable/base_table.go

@@ -35,6 +35,8 @@ import (
 type UniqueID = typeutil.UniqueID
 
 const (
+	DefaultMilvusYaml           = "milvus.yaml"
+	DefaultEasyloggingYaml      = "easylogging.yaml"
 	DefaultMinioHost            = "localhost"
 	DefaultMinioPort            = "9000"
 	DefaultMinioAccessKey       = "minioadmin"
@@ -49,7 +51,7 @@ const (
 	DefaultEnvPrefix            = "milvus"
 )
 
-var DefaultYaml = "milvus.yaml"
+var defaultYaml = DefaultMilvusYaml
 
 // Base abstracts BaseTable
 // TODO: it's never used, consider to substitute BaseTable or to remove it
@@ -79,7 +81,7 @@ type BaseTable struct {
 // GlobalInitWithYaml should be called only in standalone and embedded Milvus.
 func (gp *BaseTable) GlobalInitWithYaml(yaml string) {
 	gp.once.Do(func() {
-		DefaultYaml = yaml
+		defaultYaml = yaml
 		gp.Init()
 	})
 }
@@ -88,7 +90,7 @@ func (gp *BaseTable) GlobalInitWithYaml(yaml string) {
 func (gp *BaseTable) Init() {
 	gp.params = memkv.NewMemoryKV()
 	gp.configDir = gp.initConfPath()
-	gp.loadFromYaml(DefaultYaml)
+	gp.loadFromYaml(defaultYaml)
 	gp.tryLoadFromEnv()
 	gp.InitLogCfg()
 }

internal/util/paramtable/base_table_test.go

@@ -22,8 +22,6 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-const defaultYaml = "milvus.yaml"
-
 var baseParams = BaseTable{}
 
 func TestMain(m *testing.M) {