Let standalone share same LogCfg with distributed components (#13332)

Signed-off-by: yudong.cai <yudong.cai@zilliz.com>

cmd/roles/roles.go

@@ -22,17 +22,11 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
-	"path"
 	"strings"
 	"sync"
 	"syscall"
 	"time"
 
-	"github.com/milvus-io/milvus/internal/proto/internalpb"
-	"github.com/milvus-io/milvus/internal/util/healthz"
-	"github.com/milvus-io/milvus/internal/util/rocksmq/server/rocksmq"
-
-	"github.com/milvus-io/milvus/internal/util/metricsinfo"
 	"go.uber.org/zap"
 
 	"github.com/milvus-io/milvus/cmd/components"
@@ -44,11 +38,15 @@ import (
 	"github.com/milvus-io/milvus/internal/logutil"
 	"github.com/milvus-io/milvus/internal/metrics"
 	"github.com/milvus-io/milvus/internal/msgstream"
+	"github.com/milvus-io/milvus/internal/proto/internalpb"
 	"github.com/milvus-io/milvus/internal/proxy"
 	"github.com/milvus-io/milvus/internal/querycoord"
 	"github.com/milvus-io/milvus/internal/querynode"
 	"github.com/milvus-io/milvus/internal/rootcoord"
+	"github.com/milvus-io/milvus/internal/util/healthz"
+	"github.com/milvus-io/milvus/internal/util/metricsinfo"
 	"github.com/milvus-io/milvus/internal/util/paramtable"
+	"github.com/milvus-io/milvus/internal/util/rocksmq/server/rocksmq"
 	"github.com/milvus-io/milvus/internal/util/trace"
 )
 
@@ -87,17 +85,6 @@ func (mr *MilvusRoles) EnvValue(env string) bool {
 	return env == "1" || env == "true"
 }
 
-func (mr *MilvusRoles) setLogConfigFilename(filename string) *log.Config {
-	paramtable.Params.Init()
-	cfg := paramtable.Params.LogConfig
-	if len(cfg.File.RootPath) != 0 {
-		cfg.File.Filename = path.Join(cfg.File.RootPath, filename)
-	} else {
-		cfg.File.Filename = ""
-	}
-	return cfg
-}
-
 func (mr *MilvusRoles) runRootCoord(ctx context.Context, localMsg bool) *components.RootCoord {
 	var rc *components.RootCoord
 	var wg sync.WaitGroup
@@ -106,7 +93,7 @@ func (mr *MilvusRoles) runRootCoord(ctx context.Context, localMsg bool) *compone
 	go func() {
 		rootcoord.Params.InitOnce()
 
-		f := setLoggerFunc(localMsg)
+		f := setLoggerFunc()
 		rootcoord.Params.SetLogConfig(f)
 		factory := newMsgFactory(localMsg)
 		var err error
@@ -135,7 +122,7 @@ func (mr *MilvusRoles) runProxy(ctx context.Context, localMsg bool, alias string
 		proxy.Params.InitAlias(alias)
 		proxy.Params.InitOnce()
 
-		f := setLoggerFunc(localMsg)
+		f := setLoggerFunc()
 		proxy.Params.SetLogConfig(f)
 		factory := newMsgFactory(localMsg)
 		var err error
@@ -163,7 +150,7 @@ func (mr *MilvusRoles) runQueryCoord(ctx context.Context, localMsg bool) *compon
 	go func() {
 		querycoord.Params.InitOnce()
 
-		f := setLoggerFunc(localMsg)
+		f := setLoggerFunc()
 		querycoord.Params.SetLogConfig(f)
 		factory := newMsgFactory(localMsg)
 		var err error
@@ -192,7 +179,7 @@ func (mr *MilvusRoles) runQueryNode(ctx context.Context, localMsg bool, alias st
 		querynode.Params.InitAlias(alias)
 		querynode.Params.InitOnce()
 
-		f := setLoggerFunc(localMsg)
+		f := setLoggerFunc()
 		querynode.Params.SetLogConfig(f)
 		factory := newMsgFactory(localMsg)
 		var err error
@@ -220,7 +207,7 @@ func (mr *MilvusRoles) runDataCoord(ctx context.Context, localMsg bool) *compone
 	go func() {
 		datacoord.Params.InitOnce()
 
-		f := setLoggerFunc(localMsg)
+		f := setLoggerFunc()
 		datacoord.Params.SetLogConfig(f)
 		factory := newMsgFactory(localMsg)
 		var err error
@@ -248,7 +235,7 @@ func (mr *MilvusRoles) runDataNode(ctx context.Context, localMsg bool, alias str
 	go func() {
 		datanode.Params.InitAlias(alias)
 		datanode.Params.InitOnce()
-		f := setLoggerFunc(localMsg)
+		f := setLoggerFunc()
 		datanode.Params.SetLogConfig(f)
 		factory := newMsgFactory(localMsg)
 		var err error
@@ -276,7 +263,7 @@ func (mr *MilvusRoles) runIndexCoord(ctx context.Context, localMsg bool) *compon
 	go func() {
 		indexcoord.Params.InitOnce()
 
-		f := setLoggerFunc(localMsg)
+		f := setLoggerFunc()
 		indexcoord.Params.SetLogConfig(f)
 		var err error
 		is, err = components.NewIndexCoord(ctx)
@@ -304,7 +291,7 @@ func (mr *MilvusRoles) runIndexNode(ctx context.Context, localMsg bool, alias st
 		indexnode.Params.InitAlias(alias)
 		indexnode.Params.InitOnce()
 
-		f := setLoggerFunc(localMsg)
+		f := setLoggerFunc()
 		indexnode.Params.SetLogConfig(f)
 		var err error
 		in, err = components.NewIndexNode(ctx)
@@ -336,19 +323,20 @@ func (mr *MilvusRoles) Run(localMsg bool, alias string) {
 
 	// only standalone enable localMsg
 	if localMsg {
-		os.Setenv(metricsinfo.DeployModeEnvKey, metricsinfo.StandaloneDeployMode)
+		if err := os.Setenv(metricsinfo.DeployModeEnvKey, metricsinfo.StandaloneDeployMode); err != nil {
-		cfg := mr.setLogConfigFilename("standalone.log")
+			log.Error("Failed to set deploy mode: ", zap.Error(err))
-		logutil.SetupLogger(cfg)
+		}
-		defer log.Sync()
 
-		err := initRocksmq()
+		paramtable.Params.Init()
-		if err != nil {
+		f := setLoggerFunc()
+		paramtable.Params.SetLogConfig(f)
+
+		if err := initRocksmq(); err != nil {
 			panic(err)
 		}
 		defer stopRocksmq()
 	} else {
-		err := os.Setenv(metricsinfo.DeployModeEnvKey, metricsinfo.ClusterDeployMode)
+		if err := os.Setenv(metricsinfo.DeployModeEnvKey, metricsinfo.ClusterDeployMode); err != nil {
-		if err != nil {
 			log.Error("Failed to set deploy mode: ", zap.Error(err))
 		}
 	}
@@ -477,14 +465,10 @@ func (mr *MilvusRoles) Run(localMsg bool, alias string) {
 	cancel()
 }
 
-func setLoggerFunc(localMsg bool) func(cfg log.Config) {
+func setLoggerFunc() func(cfg log.Config) {
-	if !localMsg {
+	return func(cfg log.Config) {
-		return func(cfg log.Config) {
+		log.Info("Set log file to ", zap.String("path", cfg.File.Filename))
-			log.Info("Set log file to ", zap.String("path", cfg.File.Filename))
+		logutil.SetupLogger(&cfg)
-			logutil.SetupLogger(&cfg)
+		defer log.Sync()
-			defer log.Sync()
-		}
 	}
-	// no need to setup logger for standalone
-	return func(cfg log.Config) {}
 }

internal/util/paramtable/basetable.go

@@ -48,9 +48,9 @@ type BaseTable struct {
 	params    *memkv.MemoryKV
 	configDir string
 
-	RoleName          string
+	RoleName   string
-	Log               log.Config
+	Log        log.Config
-	LogConfigFunction func(log.Config)
+	LogCfgFunc func(log.Config)
 }
 
 func (gp *BaseTable) Init() {
@@ -451,7 +451,7 @@ func (gp *BaseTable) InitLogCfg() {
 }
 
 func (gp *BaseTable) SetLogConfig(f func(log.Config)) {
-	gp.LogConfigFunction = f
+	gp.LogCfgFunc = f
 }
 
 func (gp *BaseTable) SetLogger(id UniqueID) {
@@ -470,7 +470,7 @@ func (gp *BaseTable) SetLogger(id UniqueID) {
 		gp.Log.File.Filename = ""
 	}
 
-	if gp.LogConfigFunction != nil {
+	if gp.LogCfgFunc != nil {
-		gp.LogConfigFunction(gp.Log)
+		gp.LogCfgFunc(gp.Log)
 	}
 }

internal/util/paramtable/param.go

@@ -17,8 +17,6 @@ import (
 	"sync"
 
 	"github.com/milvus-io/milvus/internal/util/metricsinfo"
-
-	"github.com/milvus-io/milvus/internal/log"
 )
 
 // Params is a package scoped variable of type BaseParamTable.
@@ -41,8 +39,6 @@ type BaseParamTable struct {
 	EtcdDataDir    string
 
 	initOnce sync.Once
-
-	LogConfig *log.Config
 }
 
 // Init is an override method of BaseTable's Init. It mainly calls the
@@ -59,7 +55,6 @@ func (p *BaseParamTable) LoadCfgToMemory() {
 	p.initEtcdConf()
 	p.initMetaRootPath()
 	p.initKvRootPath()
-	p.initLogCfg()
 }
 
 func (p *BaseParamTable) initEtcdConf() {
@@ -120,24 +115,3 @@ func (p *BaseParamTable) initKvRootPath() {
 	}
 	p.KvRootPath = rootPath + "/" + subPath
 }
-
-func (p *BaseParamTable) initLogCfg() {
-	p.LogConfig = &log.Config{}
-	format, err := p.Load("log.format")
-	if err != nil {
-		panic(err)
-	}
-	p.LogConfig.Format = format
-	level, err := p.Load("log.level")
-	if err != nil {
-		panic(err)
-	}
-	p.LogConfig.Level = level
-	p.LogConfig.File.MaxSize = p.ParseInt("log.file.maxSize")
-	p.LogConfig.File.MaxBackups = p.ParseInt("log.file.maxBackups")
-	p.LogConfig.File.MaxDays = p.ParseInt("log.file.maxAge")
-	p.LogConfig.File.RootPath, err = p.Load("log.file.rootPath")
-	if err != nil {
-		panic(err)
-	}
-}