diff --git a/internal/proxy/accesslog/access_log.go b/internal/proxy/accesslog/access_log.go index 6cb7961b70..6a56f3029b 100644 --- a/internal/proxy/accesslog/access_log.go +++ b/internal/proxy/accesslog/access_log.go @@ -72,7 +72,12 @@ func InitAccessLogger(logCfg *paramtable.AccessLogConfig, minioCfg *paramtable.M return nil, err } - writeSyncer = zapcore.AddSync(lg) + if logCfg.CacheSize.GetAsInt() > 0 { + blg := NewCacheLogger(lg, logCfg.CacheSize.GetAsInt()) + writeSyncer = zapcore.AddSync(blg) + } else { + writeSyncer = zapcore.AddSync(lg) + } } else { stdout, _, err := zap.Open([]string{"stdout"}...) if err != nil { diff --git a/internal/proxy/accesslog/log_writer.go b/internal/proxy/accesslog/log_writer.go index 4c7b8b57b6..8cee3b643d 100644 --- a/internal/proxy/accesslog/log_writer.go +++ b/internal/proxy/accesslog/log_writer.go @@ -17,8 +17,10 @@ package accesslog import ( + "bufio" "context" "fmt" + "io" "os" "path" "sync" @@ -37,15 +39,34 @@ var ( timeFormat = ".2006-01-02T15-04-05.000" ) +type CacheLogger struct { + mu sync.Mutex + writer io.Writer +} + +func NewCacheLogger(writer io.Writer, cacheSize int) *CacheLogger { + return &CacheLogger{ + writer: bufio.NewWriterSize(writer, cacheSize), + } +} + +func (l *CacheLogger) Write(p []byte) (n int, err error) { + l.mu.Lock() + defer l.mu.Unlock() + + return l.writer.Write(p) +} + // a rotated file logger for zap.log and could upload sealed log file to minIO type RotateLogger struct { // local path is the path to save log before update to minIO // use os.TempDir()/accesslog if empty localPath string fileName string - // the interval time of update log to minIO + // the time interval of rotate and update log to minIO + // only used when minIO enable rotatedTime int64 - // the max size(Mb) of log file + // the max size(MB) of log file // if local file large than maxSize will update immediately // close if empty(zero) maxSize int @@ -77,6 +98,7 @@ func NewRotateLogger(logCfg *paramtable.AccessLogConfig, minioCfg *paramtable.Mi if logCfg.MinioEnable.GetAsBool() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() + log.Debug("remtepath", zap.Any("remote", logCfg.RemotePath.GetValue())) log.Debug("maxBackups", zap.Any("maxBackups", logCfg.MaxBackups.GetValue())) handler, err := NewMinioHandler(ctx, minioCfg, logCfg.RemotePath.GetValue(), logCfg.MaxBackups.GetAsInt()) @@ -218,6 +240,7 @@ func (l *RotateLogger) closeFile() error { return err } +// Remove old log when log num over maxBackups func (l *RotateLogger) millRunOnce() error { files, err := l.oldLogFiles() if err != nil { diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 4dec480814..d798bd9f8e 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -861,6 +861,7 @@ type AccessLogConfig struct { LocalPath ParamItem `refreshable:"false"` Filename ParamItem `refreshable:"false"` MaxSize ParamItem `refreshable:"false"` + CacheSize ParamItem `refreshable:"false"` RotatedTime ParamItem `refreshable:"false"` MaxBackups ParamItem `refreshable:"false"` RemotePath ParamItem `refreshable:"false"` @@ -1073,6 +1074,14 @@ please adjust in embedded Milvus: false`, } p.AccessLog.MaxSize.Init(base.mgr) + p.AccessLog.CacheSize = ParamItem{ + Key: "proxy.accessLog.maxSize", + Version: "2.3.2", + DefaultValue: "10240", + Doc: "Size of log of memory cache, in B", + } + p.AccessLog.CacheSize.Init(base.mgr) + p.AccessLog.MaxBackups = ParamItem{ Key: "proxy.accessLog.maxBackups", Version: "2.2.0",