From 27525d57cc6ff34407201ba3cb756de7078430ed Mon Sep 17 00:00:00 2001 From: Zhen Ye Date: Sun, 4 Jan 2026 14:45:23 +0800 Subject: [PATCH] enhance: add glog sink to transfer cgo log into zap (#46721) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit issue: #45640 - After async logging, the C log and go log has no order promise, meanwhile the C log format is not consistent with Go Log; so we close the output of glog, just forward the log result operation into Go side which will be handled by the async zap logger. - Use CGO to filter all cgo logging and promise the order between c log and go log. - Also fix the metric name, add new metric to count the logging. - TODO: after woodpecker use the logger of milvus, we can add bigger buffer for logging. - Core invariant: all C (glog) and Go logs must be routed through the same zap async pipeline so ordering and formatting are preserved; this PR ensures every glog emission is captured and forwarded to zap before any async buffering diverges the outputs. - Logic removed/simplified: direct glog outputs and hard stdout/stderr/log_dir settings are disabled (configs/glog.conf and flags in internal/core/src/config/ConfigKnowhere.cpp) because they are redundant once a single zap sink handles all logs; logging metrics were simplified from per-length/volatile gauges to totalized counters (pkg/metrics/logging_metrics.go & pkg/log/*), removing duplicate length-tracking and making accounting consistent. - No data loss or behavior regression (concrete code paths): Google logging now adds a GoZapSink (internal/core/src/common/logging_c.h, logging_c.cpp) that calls the exported CGO bridge goZapLogExt (internal/util/cgo/logging/logging.go). Go side uses C.GoStringN/C.GoString to capture full message and file, maps glog severities to zapcore levels, preserves caller info, and writes via the existing zap async core (same write path used by Go logs). The C++ send() trims glog's trailing newline and forwards exact buffers/lengths, so message content, file, line, and severity are preserved and serialized through the same async writer—no log entries are dropped or reordered relative to Go logs. - Capability added (where it takes effect): a CGO bridge that forwards glog into zap—new Go-exported function goZapLogExt (internal/util/cgo/logging/logging.go), a GoZapSink in C++ that forwards glog sends (internal/core/src/common/logging_c.h/.cpp), and blank imports of the cgo initializer across multiple packages (various internal/* files) to ensure the bridge is registered early so all C logs are captured. Signed-off-by: chyezh --- cmd/embedded/embedded.go | 1 + configs/glog.conf | 5 -- internal/core/CMakeLists.txt | 4 + internal/core/src/common/logging_c.cpp | 62 ++++++++++++++ internal/core/src/common/logging_c.h | 29 +++++++ internal/core/src/config/ConfigKnowhere.cpp | 10 +++ internal/datanode/index/init_segcore.go | 1 + internal/proxy/cgo_util.go | 1 + internal/querynodev2/server.go | 1 + internal/storagev2/packed/ffi_common.go | 1 + internal/util/analyzecgowrapper/analyze.go | 1 + .../util/analyzer/canalyzer/c_analyzer.go | 1 + internal/util/cgo/futures.go | 1 + internal/util/cgo/logging/logging.go | 84 +++++++++++++++++++ internal/util/cgo/logging/logging_test.go | 32 +++++++ internal/util/cgoconverter/bytes_converter.go | 1 + internal/util/indexcgowrapper/index.go | 1 + .../indexparamcheck/vector_index_checker.go | 1 + internal/util/initcore/init_core.go | 1 + internal/util/metrics/c_registry.go | 1 + internal/util/segcore/segcore_init.go | 4 + internal/util/textmatch/phrase_match.go | 1 + internal/util/vecindexmgr/vector_index_mgr.go | 1 + pkg/log/zap_async_buffered_write_core.go | 22 +++-- pkg/metrics/logging_metrics.go | 66 ++++++++++----- 25 files changed, 298 insertions(+), 35 deletions(-) create mode 100644 internal/core/src/common/logging_c.cpp create mode 100644 internal/core/src/common/logging_c.h create mode 100644 internal/util/cgo/logging/logging.go create mode 100644 internal/util/cgo/logging/logging_test.go diff --git a/cmd/embedded/embedded.go b/cmd/embedded/embedded.go index 34b979a612..311c97edd9 100644 --- a/cmd/embedded/embedded.go +++ b/cmd/embedded/embedded.go @@ -22,6 +22,7 @@ import ( "os" "github.com/milvus-io/milvus/cmd/milvus" + _ "github.com/milvus-io/milvus/internal/util/cgo" ) //export startEmbedded diff --git a/configs/glog.conf b/configs/glog.conf index c2874d892f..a4b9569fd7 100644 --- a/configs/glog.conf +++ b/configs/glog.conf @@ -1,10 +1,5 @@ -# if true, only log to stdout ---logtostdout=true ---logtostderr=false ---alsologtostderr=false # `INFO``, ``WARNING``, ``ERROR``, and ``FATAL`` are 0, 1, 2, and 3 --minloglevel=0 ---log_dir=/var/lib/milvus/logs/ # using vlog to implement debug and trace log # if set vmodule to 5, open debug level # if set vmodule to 6, open trace level diff --git a/internal/core/CMakeLists.txt b/internal/core/CMakeLists.txt index 4647026835..4344c05181 100644 --- a/internal/core/CMakeLists.txt +++ b/internal/core/CMakeLists.txt @@ -25,6 +25,10 @@ add_definitions(-DELPP_THREAD_SAFE) set(CMAKE_POSITION_INDEPENDENT_CODE ON) message( STATUS "Building using CMake version: ${CMAKE_VERSION}" ) +if ( BUILD_UNIT_TEST STREQUAL "ON" ) + add_definitions(-DWITHOUT_GO_LOGGING) +endif() + if ( MILVUS_GPU_VERSION ) add_definitions(-DMILVUS_GPU_VERSION) endif () diff --git a/internal/core/src/common/logging_c.cpp b/internal/core/src/common/logging_c.cpp new file mode 100644 index 0000000000..e79bce3918 --- /dev/null +++ b/internal/core/src/common/logging_c.cpp @@ -0,0 +1,62 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "logging_c.h" + +#ifdef WITHOUT_GO_LOGGING + +// Empty implementation when there's no go logging implementation. +void +goZapLogExt( + int severity, const char* file, int line, const char* msg, int msg_len) { +} + +#elif defined(__APPLE__) + +// Go export function. +// will be implemented in github.com/milvus-io/milvus/internal/util/cgo/logging +// macOS linker requires weak_import to allow unresolved symbols. +extern "C" void +goZapLogExt( + int severity, const char* file, int line, const char* msg, int msg_len) { +} +__attribute__((weak_import)); + +#else + +// Go export function. +// will be implemented in github.com/milvus-io/milvus/internal/util/cgo/logging +extern "C" void +goZapLogExt( + int severity, const char* file, int line, const char* msg, int msg_len); + +#endif + +void +GoZapSink::send(google::LogSeverity severity, + const char* full_filename, + const char* base_filename, + int line, + const struct tm*, + const char* message, + size_t message_len) { + // remove the '\n' added by glog + int len = static_cast(message_len); + if (len > 0 && message[len - 1] == '\n') { + len--; + } + goZapLogExt(static_cast(severity), full_filename, line, message, len); +}; diff --git a/internal/core/src/common/logging_c.h b/internal/core/src/common/logging_c.h new file mode 100644 index 0000000000..5bf9210b7a --- /dev/null +++ b/internal/core/src/common/logging_c.h @@ -0,0 +1,29 @@ +// Licensed to the LF AI& Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include + +class GoZapSink : public google::LogSink { + void + send(google::LogSeverity severity, + const char* full_filename, + const char* base_filename, + int line, + const struct tm*, + const char* message, + size_t message_len) override; +}; diff --git a/internal/core/src/config/ConfigKnowhere.cpp b/internal/core/src/config/ConfigKnowhere.cpp index 3e919050bd..dd41ba7568 100644 --- a/internal/core/src/config/ConfigKnowhere.cpp +++ b/internal/core/src/config/ConfigKnowhere.cpp @@ -22,11 +22,14 @@ #include "log/Log.h" #include "knowhere/comp/knowhere_config.h" #include "knowhere/version.h" +#include "common/logging_c.h" namespace milvus::config { std::once_flag init_knowhere_once_; +static GoZapSink g_sink; + void KnowhereInitImpl(const char* conf_file) { auto init = [&]() { @@ -35,6 +38,13 @@ KnowhereInitImpl(const char* conf_file) { knowhere::KnowhereConfig::ShowVersion(); if (!google::IsGoogleLoggingInitialized()) { google::InitGoogleLogging("milvus"); + google::AddLogSink(&g_sink); + + // log is catched by zap, so we don't need to log to stderr/stdout/files anymore. + FLAGS_logtostdout = false; + FLAGS_logtostderr = false; + FLAGS_alsologtostderr = false; + FLAGS_log_dir = ""; } #ifdef EMBEDDED_MILVUS diff --git a/internal/datanode/index/init_segcore.go b/internal/datanode/index/init_segcore.go index 59b9edaba3..47261244d9 100644 --- a/internal/datanode/index/init_segcore.go +++ b/internal/datanode/index/init_segcore.go @@ -31,6 +31,7 @@ import ( "path" "unsafe" + _ "github.com/milvus-io/milvus/internal/util/cgo" "github.com/milvus-io/milvus/internal/util/initcore" "github.com/milvus-io/milvus/internal/util/pathutil" "github.com/milvus-io/milvus/pkg/v2/util/hardware" diff --git a/internal/proxy/cgo_util.go b/internal/proxy/cgo_util.go index 910044dd61..2b9ab7c693 100644 --- a/internal/proxy/cgo_util.go +++ b/internal/proxy/cgo_util.go @@ -32,6 +32,7 @@ import ( "go.uber.org/zap" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + _ "github.com/milvus-io/milvus/internal/util/cgo" "github.com/milvus-io/milvus/pkg/v2/log" "github.com/milvus-io/milvus/pkg/v2/util/conc" "github.com/milvus-io/milvus/pkg/v2/util/hardware" diff --git a/internal/querynodev2/server.go b/internal/querynodev2/server.go index 052ab3a20b..d4ae296086 100644 --- a/internal/querynodev2/server.go +++ b/internal/querynodev2/server.go @@ -55,6 +55,7 @@ import ( "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/types" "github.com/milvus-io/milvus/internal/util/analyzer" + _ "github.com/milvus-io/milvus/internal/util/cgo" "github.com/milvus-io/milvus/internal/util/dependency" "github.com/milvus-io/milvus/internal/util/fileresource" "github.com/milvus-io/milvus/internal/util/hookutil" diff --git a/internal/storagev2/packed/ffi_common.go b/internal/storagev2/packed/ffi_common.go index 56112e7a53..2fc382b41c 100644 --- a/internal/storagev2/packed/ffi_common.go +++ b/internal/storagev2/packed/ffi_common.go @@ -15,6 +15,7 @@ import ( "strconv" "unsafe" + _ "github.com/milvus-io/milvus/internal/util/cgo" "github.com/milvus-io/milvus/pkg/v2/proto/indexpb" ) diff --git a/internal/util/analyzecgowrapper/analyze.go b/internal/util/analyzecgowrapper/analyze.go index c5dae07208..d539d07355 100644 --- a/internal/util/analyzecgowrapper/analyze.go +++ b/internal/util/analyzecgowrapper/analyze.go @@ -32,6 +32,7 @@ import ( "go.uber.org/zap" "google.golang.org/protobuf/proto" + _ "github.com/milvus-io/milvus/internal/util/cgo" "github.com/milvus-io/milvus/pkg/v2/log" "github.com/milvus-io/milvus/pkg/v2/proto/clusteringpb" ) diff --git a/internal/util/analyzer/canalyzer/c_analyzer.go b/internal/util/analyzer/canalyzer/c_analyzer.go index ae312fb45d..de32b130b4 100644 --- a/internal/util/analyzer/canalyzer/c_analyzer.go +++ b/internal/util/analyzer/canalyzer/c_analyzer.go @@ -12,6 +12,7 @@ import ( "unsafe" "github.com/milvus-io/milvus/internal/util/analyzer/interfaces" + _ "github.com/milvus-io/milvus/internal/util/cgo" ) var _ interfaces.Analyzer = (*CAnalyzer)(nil) diff --git a/internal/util/cgo/futures.go b/internal/util/cgo/futures.go index bdc770e5d1..22bc405b98 100644 --- a/internal/util/cgo/futures.go +++ b/internal/util/cgo/futures.go @@ -25,6 +25,7 @@ import ( "github.com/cockroachdb/errors" + _ "github.com/milvus-io/milvus/internal/util/cgo/logging" "github.com/milvus-io/milvus/pkg/v2/util/merr" ) diff --git a/internal/util/cgo/logging/logging.go b/internal/util/cgo/logging/logging.go new file mode 100644 index 0000000000..f3a21b2d54 --- /dev/null +++ b/internal/util/cgo/logging/logging.go @@ -0,0 +1,84 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logging + +/* +extern void goZapLogExt(int severity, + char* file, + int line, + char* msg, + int msg_len); +*/ +import "C" + +import ( + "time" + + "go.uber.org/zap/zapcore" + + "github.com/milvus-io/milvus/pkg/v2/log" + "github.com/milvus-io/milvus/pkg/v2/metrics" +) + +const cgoLoggerName = "CGO" + +//export goZapLogExt +func goZapLogExt(sev C.int, + file *C.char, + line C.int, + msg *C.char, + msgLen C.int, +) { + lv := mapGlogSeverity(int(sev)) + if !log.L().Core().Enabled(lv) { + return + } + ent := zapcore.Entry{ + Level: mapGlogSeverity(int(sev)), + Time: time.Now(), + LoggerName: cgoLoggerName, + Message: C.GoStringN(msg, msgLen), + Caller: zapcore.EntryCaller{ + Defined: true, + File: C.GoString(file), + Line: int(line), + }, + } + if ce := log.L().Core().Check(ent, nil); ce != nil { + metrics.LoggingCGOWriteTotal.Inc() + metrics.LoggingCGOWriteBytes.Add(float64(msgLen)) + ce.Write() + } +} + +func mapGlogSeverity(s int) zapcore.Level { + switch s { + case 0: // GLOG_INFO + return zapcore.InfoLevel + case 1: // GLOG_WARNING + return zapcore.WarnLevel + case 2: // GLOG_ERROR + return zapcore.ErrorLevel + case 3: // GLOG_FATAL + // glog fatal will call std::abort, + // zap will call os.Exit(1), + // we don't want to double exit, so we use error level instead + return zapcore.ErrorLevel + default: + return zapcore.InfoLevel + } +} diff --git a/internal/util/cgo/logging/logging_test.go b/internal/util/cgo/logging/logging_test.go new file mode 100644 index 0000000000..8503532431 --- /dev/null +++ b/internal/util/cgo/logging/logging_test.go @@ -0,0 +1,32 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logging + +import ( + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" +) + +func TestLogging(t *testing.T) { + require.Equal(t, zapcore.InfoLevel, mapGlogSeverity(0)) + require.Equal(t, zapcore.WarnLevel, mapGlogSeverity(1)) + require.Equal(t, zapcore.ErrorLevel, mapGlogSeverity(2)) + require.Equal(t, zapcore.ErrorLevel, mapGlogSeverity(3)) + require.Equal(t, zapcore.InfoLevel, mapGlogSeverity(4)) +} diff --git a/internal/util/cgoconverter/bytes_converter.go b/internal/util/cgoconverter/bytes_converter.go index a3d1cbedff..5c81d58b1f 100644 --- a/internal/util/cgoconverter/bytes_converter.go +++ b/internal/util/cgoconverter/bytes_converter.go @@ -10,6 +10,7 @@ import ( "sync/atomic" "unsafe" + _ "github.com/milvus-io/milvus/internal/util/cgo" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) diff --git a/internal/util/indexcgowrapper/index.go b/internal/util/indexcgowrapper/index.go index edaffcb6a1..6b95b113fe 100644 --- a/internal/util/indexcgowrapper/index.go +++ b/internal/util/indexcgowrapper/index.go @@ -23,6 +23,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/internal/storage" + _ "github.com/milvus-io/milvus/internal/util/cgo" "github.com/milvus-io/milvus/internal/util/segcore" "github.com/milvus-io/milvus/pkg/v2/log" "github.com/milvus-io/milvus/pkg/v2/proto/cgopb" diff --git a/internal/util/indexparamcheck/vector_index_checker.go b/internal/util/indexparamcheck/vector_index_checker.go index af50963478..12ed045720 100644 --- a/internal/util/indexparamcheck/vector_index_checker.go +++ b/internal/util/indexparamcheck/vector_index_checker.go @@ -18,6 +18,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + _ "github.com/milvus-io/milvus/internal/util/cgo" "github.com/milvus-io/milvus/internal/util/vecindexmgr" "github.com/milvus-io/milvus/pkg/v2/common" "github.com/milvus-io/milvus/pkg/v2/proto/indexcgopb" diff --git a/internal/util/initcore/init_core.go b/internal/util/initcore/init_core.go index 1a943aa377..7da887ae3e 100644 --- a/internal/util/initcore/init_core.go +++ b/internal/util/initcore/init_core.go @@ -41,6 +41,7 @@ import ( "go.uber.org/zap" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" + _ "github.com/milvus-io/milvus/internal/util/cgo" "github.com/milvus-io/milvus/internal/util/hookutil" "github.com/milvus-io/milvus/internal/util/pathutil" "github.com/milvus-io/milvus/pkg/v2/log" diff --git a/internal/util/metrics/c_registry.go b/internal/util/metrics/c_registry.go index e37ce88056..93ec25c56b 100644 --- a/internal/util/metrics/c_registry.go +++ b/internal/util/metrics/c_registry.go @@ -43,6 +43,7 @@ import ( "golang.org/x/exp/maps" "google.golang.org/protobuf/proto" + _ "github.com/milvus-io/milvus/internal/util/cgo" "github.com/milvus-io/milvus/pkg/v2/log" ) diff --git a/internal/util/segcore/segcore_init.go b/internal/util/segcore/segcore_init.go index b05e056a69..91a8f29902 100644 --- a/internal/util/segcore/segcore_init.go +++ b/internal/util/segcore/segcore_init.go @@ -1,5 +1,9 @@ package segcore +import ( + _ "github.com/milvus-io/milvus/internal/util/cgo" +) + /* #cgo pkg-config: milvus_core diff --git a/internal/util/textmatch/phrase_match.go b/internal/util/textmatch/phrase_match.go index 2d3229550e..2c7ce60a8d 100644 --- a/internal/util/textmatch/phrase_match.go +++ b/internal/util/textmatch/phrase_match.go @@ -11,6 +11,7 @@ import ( "fmt" "unsafe" + _ "github.com/milvus-io/milvus/internal/util/cgo" "github.com/milvus-io/milvus/pkg/v2/log" "github.com/milvus-io/milvus/pkg/v2/util/merr" ) diff --git a/internal/util/vecindexmgr/vector_index_mgr.go b/internal/util/vecindexmgr/vector_index_mgr.go index 36a17fcfec..8fbc850c00 100644 --- a/internal/util/vecindexmgr/vector_index_mgr.go +++ b/internal/util/vecindexmgr/vector_index_mgr.go @@ -31,6 +31,7 @@ import ( "unsafe" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + _ "github.com/milvus-io/milvus/internal/util/cgo" "github.com/milvus-io/milvus/pkg/v2/log" ) diff --git a/pkg/log/zap_async_buffered_write_core.go b/pkg/log/zap_async_buffered_write_core.go index 114de4cb7d..5ae73be149 100644 --- a/pkg/log/zap_async_buffered_write_core.go +++ b/pkg/log/zap_async_buffered_write_core.go @@ -130,10 +130,9 @@ func (s *asyncTextIOCore) Write(ent zapcore.Entry, fields []zapcore.Field) error } select { case s.pending <- entry: - metrics.LoggingPendingWriteLength.Inc() - metrics.LoggingPendingWriteBytes.Add(float64(length)) + metrics.LoggingPendingWriteTotal.Inc() case <-writeDroppedTimeout: - metrics.LoggingDroppedWrites.Inc() + metrics.LoggingDroppedWriteTotal.Inc() // drop the entry if the write is dropped due to timeout buf.Free() } @@ -165,15 +164,20 @@ func (s *asyncTextIOCore) background() { // consumeEntry write the entry to the underlying buffered write syncer and free the buffer. func (s *asyncTextIOCore) consumeEntry(ent *entryItem) { length := ent.buf.Len() - metrics.LoggingPendingWriteLength.Dec() - metrics.LoggingPendingWriteBytes.Sub(float64(length)) + metrics.LoggingPendingWriteTotal.Dec() + writes := s.getWriteBytes(ent) if _, err := s.bws.Write(writes); err != nil { - metrics.LoggingIOFailure.Inc() + metrics.LoggingIOFailureTotal.Inc() + } else { + metrics.LoggingWriteTotal.Inc() + metrics.LoggingWriteBytes.Add(float64(length)) } ent.buf.Free() if ent.level > zapcore.ErrorLevel { - s.bws.Sync() + if err := s.bws.Sync(); err != nil { + metrics.LoggingIOFailureTotal.Inc() + } } } @@ -186,7 +190,7 @@ func (s *asyncTextIOCore) getWriteBytes(ent *entryItem) []byte { if length > s.maxBytesPerLog { // truncate the write if it exceeds the max bytes per log - metrics.LoggingTruncatedWrites.Inc() + metrics.LoggingTruncatedWriteTotal.Inc() metrics.LoggingTruncatedWriteBytes.Add(float64(length - s.maxBytesPerLog)) end := writes[length-1] @@ -211,7 +215,7 @@ func (s *asyncTextIOCore) flushPendingWriteWithTimeout() { func (s *asyncTextIOCore) flushAllPendingWrites(done chan struct{}) { defer func() { if err := s.bws.Stop(); err != nil { - metrics.LoggingIOFailure.Inc() + metrics.LoggingIOFailureTotal.Inc() } close(done) }() diff --git a/pkg/metrics/logging_metrics.go b/pkg/metrics/logging_metrics.go index db5e3ff4cd..d520f2879c 100644 --- a/pkg/metrics/logging_metrics.go +++ b/pkg/metrics/logging_metrics.go @@ -29,57 +29,81 @@ const ( var ( LoggingMetricsRegisterOnce sync.Once - LoggingPendingWriteLength = prometheus.NewGauge(prometheus.GaugeOpts{ + LoggingPendingWriteTotal = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: milvusNamespace, Subsystem: loggingMetricSubsystem, - Name: "pending_write_length", + Name: "pending_write_total", Help: "The length of pending writes in the logging buffer", }) - LoggingPendingWriteBytes = prometheus.NewGauge(prometheus.GaugeOpts{ + LoggingTruncatedWriteTotal = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: milvusNamespace, Subsystem: loggingMetricSubsystem, - Name: "pending_write_bytes", - Help: "The total bytes of pending writes in the logging buffer", - }) - - LoggingTruncatedWrites = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: milvusNamespace, - Subsystem: loggingMetricSubsystem, - Name: "truncated_writes", + Name: "truncated_write_total", Help: "The number of truncated writes due to exceeding the max bytes per log", }) - LoggingTruncatedWriteBytes = prometheus.NewGauge(prometheus.GaugeOpts{ + LoggingTruncatedWriteBytes = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: milvusNamespace, Subsystem: loggingMetricSubsystem, Name: "truncated_write_bytes", Help: "The total bytes of truncated writes due to exceeding the max bytes per log", }) - LoggingDroppedWrites = prometheus.NewGauge(prometheus.GaugeOpts{ + LoggingDroppedWriteTotal = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: milvusNamespace, Subsystem: loggingMetricSubsystem, - Name: "dropped_writes", + Name: "dropped_write_total", Help: "The number of dropped writes due to buffer full or write timeout", }) - LoggingIOFailure = prometheus.NewGauge(prometheus.GaugeOpts{ + LoggingIOFailureTotal = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: milvusNamespace, Subsystem: loggingMetricSubsystem, - Name: "io_failures", + Name: "io_failure_total", Help: "The number of IO failures due to underlying write syncer is blocked or write timeout", }) + + LoggingWriteTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: milvusNamespace, + Subsystem: loggingMetricSubsystem, + Name: "write_total", + Help: "The total number of writes", + }) + + LoggingWriteBytes = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: milvusNamespace, + Subsystem: loggingMetricSubsystem, + Name: "write_bytes", + Help: "The total bytes of written logs", + }) + + LoggingCGOWriteTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: milvusNamespace, + Subsystem: loggingMetricSubsystem, + Name: "cgo_write_total", + Help: "The total number of CGO writes", + }) + + LoggingCGOWriteBytes = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: milvusNamespace, + Subsystem: loggingMetricSubsystem, + Name: "cgo_write_bytes", + Help: "The total bytes of CGO write logs, the bytes is calculated before encoding, only considers the length of the message, so the actual bytes may be greater than the value", + }) ) // RegisterLoggingMetrics registers logging metrics func RegisterLoggingMetrics(registry *prometheus.Registry) { LoggingMetricsRegisterOnce.Do(func() { - registry.MustRegister(LoggingPendingWriteLength) - registry.MustRegister(LoggingPendingWriteBytes) - registry.MustRegister(LoggingTruncatedWrites) + registry.MustRegister(LoggingPendingWriteTotal) + registry.MustRegister(LoggingTruncatedWriteTotal) registry.MustRegister(LoggingTruncatedWriteBytes) - registry.MustRegister(LoggingDroppedWrites) - registry.MustRegister(LoggingIOFailure) + registry.MustRegister(LoggingDroppedWriteTotal) + registry.MustRegister(LoggingIOFailureTotal) + registry.MustRegister(LoggingWriteTotal) + registry.MustRegister(LoggingWriteBytes) + registry.MustRegister(LoggingCGOWriteTotal) + registry.MustRegister(LoggingCGOWriteBytes) }) }