From d38adda8c23cfb05743c01decc1e23c5a2581ae6 Mon Sep 17 00:00:00 2001 From: Xiaofan <83447078+xiaofan-luan@users.noreply.github.com> Date: Sun, 26 Sep 2021 21:18:11 +0800 Subject: [PATCH] Rename log file based on node id (#8517) Signed-off-by: xiaofan-luan --- internal/datacoord/param_table.go | 30 +--------------------- internal/datanode/data_node.go | 1 + internal/datanode/param_table.go | 27 +------------------ internal/indexcoord/param_table.go | 28 +------------------- internal/indexnode/indexnode.go | 1 + internal/indexnode/param_table.go | 28 +------------------- internal/proxy/param_table.go | 28 +------------------- internal/proxy/proxy.go | 1 + internal/querycoord/param_table.go | 29 +-------------------- internal/querynode/param_table.go | 30 +--------------------- internal/querynode/query_node.go | 1 + internal/rootcoord/param_table.go | 28 +------------------- internal/util/paramtable/basetable.go | 28 ++++++++++++++++++++ internal/util/paramtable/basetable_test.go | 14 ++++++++++ 14 files changed, 54 insertions(+), 220 deletions(-) diff --git a/internal/datacoord/param_table.go b/internal/datacoord/param_table.go index 15463b575b..59fddb7578 100644 --- a/internal/datacoord/param_table.go +++ b/internal/datacoord/param_table.go @@ -12,14 +12,10 @@ package datacoord import ( - "path" - "strconv" "strings" "sync" "time" - "github.com/milvus-io/milvus/internal/log" - "github.com/milvus-io/milvus/internal/util/paramtable" ) @@ -62,8 +58,6 @@ type ParamTable struct { CreatedTime time.Time UpdatedTime time.Time - - Log log.Config } var Params ParamTable @@ -242,29 +236,7 @@ func (p *ParamTable) initDataCoordSubscriptionName() { } func (p *ParamTable) initLogCfg() { - p.Log = log.Config{} - format, err := p.Load("log.format") - if err != nil { - panic(err) - } - p.Log.Format = format - level, err := p.Load("log.level") - if err != nil { - panic(err) - } - p.Log.Level = level - p.Log.File.MaxSize = p.ParseInt("log.file.maxSize") - p.Log.File.MaxBackups = p.ParseInt("log.file.maxBackups") - p.Log.File.MaxDays = p.ParseInt("log.file.maxAge") - rootPath, err := p.Load("log.file.rootPath") - if err != nil { - panic(err) - } - if len(rootPath) != 0 { - p.Log.File.Filename = path.Join(rootPath, "datacoord-"+strconv.FormatInt(p.NodeID, 10)+".log") - } else { - p.Log.File.Filename = "" - } + p.InitLogCfg("datacoord", 0) } func (p *ParamTable) initFlushStreamPosSubPath() { diff --git a/internal/datanode/data_node.go b/internal/datanode/data_node.go index 471a86ae50..cb15734fdf 100644 --- a/internal/datanode/data_node.go +++ b/internal/datanode/data_node.go @@ -157,6 +157,7 @@ func (node *DataNode) Register() error { go node.StartWatchChannels(node.ctx) Params.initMsgChannelSubName() + Params.initLogCfg() log.Debug("DataNode Init", zap.String("MsgChannelSubName", Params.MsgChannelSubName), ) diff --git a/internal/datanode/param_table.go b/internal/datanode/param_table.go index d7eaefe10d..2e09e01046 100644 --- a/internal/datanode/param_table.go +++ b/internal/datanode/param_table.go @@ -18,7 +18,6 @@ import ( "sync" "time" - "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/util/paramtable" ) @@ -35,7 +34,6 @@ type ParamTable struct { FlushInsertBufferSize int64 InsertBinlogRootPath string StatsBinlogRootPath string - Log log.Config Alias string // Different datanode in one machine // === DataNode External Components Configs === @@ -102,7 +100,6 @@ func (p *ParamTable) Init() { p.initFlushInsertBufferSize() p.initInsertBinlogRootPath() p.initStatsBinlogRootPath() - p.initLogCfg() // === DataNode External Components Configs === // --- Pulsar --- @@ -279,27 +276,5 @@ func (p *ParamTable) initMinioBucketName() { } func (p *ParamTable) initLogCfg() { - p.Log = log.Config{} - format, err := p.Load("log.format") - if err != nil { - panic(err) - } - p.Log.Format = format - level, err := p.Load("log.level") - if err != nil { - panic(err) - } - p.Log.Level = level - p.Log.File.MaxSize = p.ParseInt("log.file.maxSize") - p.Log.File.MaxBackups = p.ParseInt("log.file.maxBackups") - p.Log.File.MaxDays = p.ParseInt("log.file.maxAge") - rootPath, err := p.Load("log.file.rootPath") - if err != nil { - panic(err) - } - if len(rootPath) != 0 { - p.Log.File.Filename = path.Join(rootPath, "datanode"+p.Alias+".log") - } else { - p.Log.File.Filename = "" - } + p.InitLogCfg("datanode", p.NodeID) } diff --git a/internal/indexcoord/param_table.go b/internal/indexcoord/param_table.go index b90c678bf0..0ea175453f 100644 --- a/internal/indexcoord/param_table.go +++ b/internal/indexcoord/param_table.go @@ -12,13 +12,11 @@ package indexcoord import ( - "path" "strconv" "strings" "sync" "time" - "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/util/paramtable" ) @@ -40,8 +38,6 @@ type ParamTable struct { CreatedTime time.Time UpdatedTime time.Time - - Log log.Config } var Params ParamTable @@ -148,27 +144,5 @@ func (pt *ParamTable) initMinioBucketName() { } func (pt *ParamTable) initLogCfg() { - pt.Log = log.Config{} - format, err := pt.Load("log.format") - if err != nil { - panic(err) - } - pt.Log.Format = format - level, err := pt.Load("log.level") - if err != nil { - panic(err) - } - pt.Log.Level = level - pt.Log.File.MaxSize = pt.ParseInt("log.file.maxSize") - pt.Log.File.MaxBackups = pt.ParseInt("log.file.maxBackups") - pt.Log.File.MaxDays = pt.ParseInt("log.file.maxAge") - rootPath, err := pt.Load("log.file.rootPath") - if err != nil { - panic(err) - } - if len(rootPath) != 0 { - pt.Log.File.Filename = path.Join(rootPath, "indexcoord.log") - } else { - pt.Log.File.Filename = "" - } + pt.InitLogCfg("indexcoord", 0) } diff --git a/internal/indexnode/indexnode.go b/internal/indexnode/indexnode.go index b15d8d71f6..0645b59d5e 100644 --- a/internal/indexnode/indexnode.go +++ b/internal/indexnode/indexnode.go @@ -106,6 +106,7 @@ func (i *IndexNode) Register() error { } i.liveCh = i.session.Init(typeutil.IndexNodeRole, Params.IP+":"+strconv.Itoa(Params.Port), false) Params.NodeID = i.session.ServerID + Params.initLogCfg() return nil } diff --git a/internal/indexnode/param_table.go b/internal/indexnode/param_table.go index c18a12d7fa..652e66fe16 100644 --- a/internal/indexnode/param_table.go +++ b/internal/indexnode/param_table.go @@ -12,7 +12,6 @@ package indexnode import ( - "fmt" "path" "strconv" "strings" @@ -53,8 +52,6 @@ type ParamTable struct { CreatedTime time.Time UpdatedTime time.Time - - Log log.Config } var Params ParamTable @@ -76,7 +73,6 @@ func (pt *ParamTable) Init() { panic(err) }*/ - pt.initLogCfg() pt.initParams() pt.initKnowhereSimdType() } @@ -170,29 +166,7 @@ func (pt *ParamTable) initMinioBucketName() { } func (pt *ParamTable) initLogCfg() { - pt.Log = log.Config{} - format, err := pt.Load("log.format") - if err != nil { - panic(err) - } - pt.Log.Format = format - level, err := pt.Load("log.level") - if err != nil { - panic(err) - } - pt.Log.Level = level - pt.Log.File.MaxSize = pt.ParseInt("log.file.maxSize") - pt.Log.File.MaxBackups = pt.ParseInt("log.file.maxBackups") - pt.Log.File.MaxDays = pt.ParseInt("log.file.maxAge") - rootPath, err := pt.Load("log.file.rootPath") - if err != nil { - panic(err) - } - if len(rootPath) != 0 { - pt.Log.File.Filename = path.Join(rootPath, fmt.Sprintf("indexnode-%s.log", pt.Alias)) - } else { - pt.Log.File.Filename = "" - } + pt.InitLogCfg("indexnode", pt.NodeID) } func (pt *ParamTable) initKnowhereSimdType() { diff --git a/internal/proxy/param_table.go b/internal/proxy/param_table.go index 3d21fba3ca..8ee3e75366 100644 --- a/internal/proxy/param_table.go +++ b/internal/proxy/param_table.go @@ -19,7 +19,6 @@ import ( "sync" "time" - "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/util/paramtable" ) @@ -37,7 +36,6 @@ type ParamTable struct { NetworkAddress string - // TODO(dragondriver): maybe using the Proxy + ProxyID as the alias is more reasonable Alias string EtcdEndpoints []string @@ -69,7 +67,6 @@ type ParamTable struct { MaxTaskNum int64 PulsarMaxMessageSize int - Log log.Config RoleName string } @@ -89,7 +86,6 @@ func (pt *ParamTable) Init() { panic(err) } - pt.initLogCfg() pt.initEtcdEndpoints() pt.initMetaRootPath() pt.initPulsarAddress() @@ -277,29 +273,7 @@ func (pt *ParamTable) initPulsarMaxMessageSize() { } func (pt *ParamTable) initLogCfg() { - pt.Log = log.Config{} - format, err := pt.Load("log.format") - if err != nil { - panic(err) - } - pt.Log.Format = format - level, err := pt.Load("log.level") - if err != nil { - panic(err) - } - pt.Log.Level = level - pt.Log.File.MaxSize = pt.ParseInt("log.file.maxSize") - pt.Log.File.MaxBackups = pt.ParseInt("log.file.maxBackups") - pt.Log.File.MaxDays = pt.ParseInt("log.file.maxAge") - rootPath, err := pt.Load("log.file.rootPath") - if err != nil { - panic(err) - } - if len(rootPath) != 0 { - pt.Log.File.Filename = path.Join(rootPath, fmt.Sprintf("proxy-%s.log", pt.Alias)) - } else { - pt.Log.File.Filename = "" - } + pt.InitLogCfg("proxy", pt.ProxyID) } func (pt *ParamTable) initRoleName() { diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index e14f2baa76..8a5acafcb4 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -100,6 +100,7 @@ func (node *Proxy) Register() error { node.session.Init(typeutil.ProxyRole, Params.NetworkAddress, false) Params.ProxyID = node.session.ServerID Params.initProxySubName() + Params.initLogCfg() return nil } diff --git a/internal/querycoord/param_table.go b/internal/querycoord/param_table.go index 3b937ca4ff..0537275ca3 100644 --- a/internal/querycoord/param_table.go +++ b/internal/querycoord/param_table.go @@ -40,7 +40,6 @@ type ParamTable struct { // timetick TimeTickChannelName string - Log log.Config RoleName string // channels @@ -81,7 +80,7 @@ func (p *ParamTable) Init() { panic(err) } - p.initLogCfg() + p.BaseTable.InitLogCfg("queryCoord", 0) p.initQueryCoordAddress() p.initRoleName() @@ -106,32 +105,6 @@ func (p *ParamTable) Init() { p.initMinioBucketName() } -func (p *ParamTable) initLogCfg() { - p.Log = log.Config{} - format, err := p.Load("log.format") - if err != nil { - panic(err) - } - p.Log.Format = format - level, err := p.Load("log.level") - if err != nil { - panic(err) - } - p.Log.Level = level - p.Log.File.MaxSize = p.ParseInt("log.file.maxSize") - p.Log.File.MaxBackups = p.ParseInt("log.file.maxBackups") - p.Log.File.MaxDays = p.ParseInt("log.file.maxAge") - rootPath, err := p.Load("log.file.rootPath") - if err != nil { - panic(err) - } - if len(rootPath) != 0 { - p.Log.File.Filename = path.Join(rootPath, fmt.Sprintf("queryCoord-%d.log", p.NodeID)) - } else { - p.Log.File.Filename = "" - } -} - func (p *ParamTable) initQueryCoordAddress() { url, err := p.Load("_QueryCoordAddress") if err != nil { diff --git a/internal/querynode/param_table.go b/internal/querynode/param_table.go index 68037e6871..c21f3097da 100644 --- a/internal/querynode/param_table.go +++ b/internal/querynode/param_table.go @@ -12,8 +12,6 @@ package querynode import ( - "fmt" - "path" "strconv" "strings" "sync" @@ -74,8 +72,6 @@ type ParamTable struct { // segcore ChunkRows int64 SimdType string - - Log log.Config } var Params ParamTable @@ -130,8 +126,6 @@ func (p *ParamTable) Init() { p.initSegcoreChunkRows() p.initKnowhereSimdType() - - p.initLogCfg() } // ---------------------------------------------------------- minio @@ -298,27 +292,5 @@ func (p *ParamTable) initKnowhereSimdType() { } func (p *ParamTable) initLogCfg() { - p.Log = log.Config{} - format, err := p.Load("log.format") - if err != nil { - panic(err) - } - p.Log.Format = format - level, err := p.Load("log.level") - if err != nil { - panic(err) - } - p.Log.Level = level - p.Log.File.MaxSize = p.ParseInt("log.file.maxSize") - p.Log.File.MaxBackups = p.ParseInt("log.file.maxBackups") - p.Log.File.MaxDays = p.ParseInt("log.file.maxAge") - rootPath, err := p.Load("log.file.rootPath") - if err != nil { - panic(err) - } - if len(rootPath) != 0 { - p.Log.File.Filename = path.Join(rootPath, fmt.Sprintf("querynode-%s.log", p.Alias)) - } else { - p.Log.File.Filename = "" - } + p.InitLogCfg("querynode", p.QueryNodeID) } diff --git a/internal/querynode/query_node.go b/internal/querynode/query_node.go index d5d4af8d94..9b93629d3b 100644 --- a/internal/querynode/query_node.go +++ b/internal/querynode/query_node.go @@ -109,6 +109,7 @@ func (node *QueryNode) Register() error { // This param needs valid QueryNodeID Params.initMsgChannelSubName() + Params.initLogCfg() return nil } diff --git a/internal/rootcoord/param_table.go b/internal/rootcoord/param_table.go index e2e62b945d..3b1023e5b9 100644 --- a/internal/rootcoord/param_table.go +++ b/internal/rootcoord/param_table.go @@ -12,12 +12,10 @@ package rootcoord import ( - "path" "strings" "sync" "time" - "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/util/paramtable" ) @@ -55,8 +53,6 @@ type ParamTable struct { CreatedTime time.Time UpdatedTime time.Time - Log log.Config - RoleName string } @@ -222,29 +218,7 @@ func (p *ParamTable) initTimeTickInterval() { } func (p *ParamTable) initLogCfg() { - p.Log = log.Config{} - format, err := p.Load("log.format") - if err != nil { - panic(err) - } - p.Log.Format = format - level, err := p.Load("log.level") - if err != nil { - panic(err) - } - p.Log.Level = level - p.Log.File.MaxSize = p.ParseInt("log.file.maxSize") - p.Log.File.MaxBackups = p.ParseInt("log.file.maxBackups") - p.Log.File.MaxDays = p.ParseInt("log.file.maxAge") - rootPath, err := p.Load("log.file.rootPath") - if err != nil { - panic(err) - } - if len(rootPath) != 0 { - p.Log.File.Filename = path.Join(rootPath, "rootcoord.log") - } else { - p.Log.File.Filename = "" - } + p.InitLogCfg("rootcoord", 0) } func (p *ParamTable) initRoleName() { diff --git a/internal/util/paramtable/basetable.go b/internal/util/paramtable/basetable.go index 340ba3f030..89b8a495a1 100644 --- a/internal/util/paramtable/basetable.go +++ b/internal/util/paramtable/basetable.go @@ -43,6 +43,8 @@ type Base interface { type BaseTable struct { params *memkv.MemoryKV configDir string + + Log log.Config } func (gp *BaseTable) Init() { @@ -413,3 +415,29 @@ func ConvertRangeToIntSlice(rangeStr, sep string) []int { } return ret } + +func (gp *BaseTable) InitLogCfg(role string, id UniqueID) { + gp.Log = log.Config{} + format, err := gp.Load("log.format") + if err != nil { + panic(err) + } + gp.Log.Format = format + level, err := gp.Load("log.level") + if err != nil { + panic(err) + } + gp.Log.Level = level + gp.Log.File.MaxSize = gp.ParseInt("log.file.maxSize") + gp.Log.File.MaxBackups = gp.ParseInt("log.file.maxBackups") + gp.Log.File.MaxDays = gp.ParseInt("log.file.maxAge") + rootPath, err := gp.Load("log.file.rootPath") + if err != nil { + panic(err) + } + if len(rootPath) != 0 { + gp.Log.File.Filename = path.Join(rootPath, role+"-"+strconv.FormatInt(id, 10)+".log") + } else { + gp.Log.File.Filename = "" + } +} diff --git a/internal/util/paramtable/basetable_test.go b/internal/util/paramtable/basetable_test.go index 257633c99c..bf9f897318 100644 --- a/internal/util/paramtable/basetable_test.go +++ b/internal/util/paramtable/basetable_test.go @@ -225,3 +225,17 @@ func Test_ConvertRangeToIntSlice(t *testing.T) { assert.Panics(t, func() { ConvertRangeToIntSlice("9,0", ",") }) }) } + +func Test_InitLogCfg(t *testing.T) { + t.Run("TestInitLogCfg", func(t *testing.T) { + baseParams.InitLogCfg("rootcoord", 0) + assert.Equal(t, baseParams.Log.File.Filename, "") + + baseParams.Save("log.file.rootPath", "/") + baseParams.InitLogCfg("rootcoord", 0) + assert.Equal(t, baseParams.Log.File.Filename, "/rootcoord-0.log") + + baseParams.InitLogCfg("datanode", 8) + assert.Equal(t, baseParams.Log.File.Filename, "/datanode-8.log") + }) +}