mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-01-07 19:31:51 +08:00
27525d57cc
issue: #45640 - After async logging, the C log and go log has no order promise, meanwhile the C log format is not consistent with Go Log; so we close the output of glog, just forward the log result operation into Go side which will be handled by the async zap logger. - Use CGO to filter all cgo logging and promise the order between c log and go log. - Also fix the metric name, add new metric to count the logging. - TODO: after woodpecker use the logger of milvus, we can add bigger buffer for logging. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> - Core invariant: all C (glog) and Go logs must be routed through the same zap async pipeline so ordering and formatting are preserved; this PR ensures every glog emission is captured and forwarded to zap before any async buffering diverges the outputs. - Logic removed/simplified: direct glog outputs and hard stdout/stderr/log_dir settings are disabled (configs/glog.conf and flags in internal/core/src/config/ConfigKnowhere.cpp) because they are redundant once a single zap sink handles all logs; logging metrics were simplified from per-length/volatile gauges to totalized counters (pkg/metrics/logging_metrics.go & pkg/log/*), removing duplicate length-tracking and making accounting consistent. - No data loss or behavior regression (concrete code paths): Google logging now adds a GoZapSink (internal/core/src/common/logging_c.h, logging_c.cpp) that calls the exported CGO bridge goZapLogExt (internal/util/cgo/logging/logging.go). Go side uses C.GoStringN/C.GoString to capture full message and file, maps glog severities to zapcore levels, preserves caller info, and writes via the existing zap async core (same write path used by Go logs). The C++ send() trims glog's trailing newline and forwards exact buffers/lengths, so message content, file, line, and severity are preserved and serialized through the same async writer—no log entries are dropped or reordered relative to Go logs. - Capability added (where it takes effect): a CGO bridge that forwards glog into zap—new Go-exported function goZapLogExt (internal/util/cgo/logging/logging.go), a GoZapSink in C++ that forwards glog sends (internal/core/src/common/logging_c.h/.cpp), and blank imports of the cgo initializer across multiple packages (various internal/* files) to ensure the bridge is registered early so all C logs are captured. <!-- end of auto-generated comment: release notes by coderabbit.ai --> Signed-off-by: chyezh <chyezh@outlook.com>
85 lines
2.1 KiB
Go
85 lines
2.1 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package logging
|
|
|
|
/*
|
|
extern void goZapLogExt(int severity,
|
|
char* file,
|
|
int line,
|
|
char* msg,
|
|
int msg_len);
|
|
*/
|
|
import "C"
|
|
|
|
import (
|
|
"time"
|
|
|
|
"go.uber.org/zap/zapcore"
|
|
|
|
"github.com/milvus-io/milvus/pkg/v2/log"
|
|
"github.com/milvus-io/milvus/pkg/v2/metrics"
|
|
)
|
|
|
|
const cgoLoggerName = "CGO"
|
|
|
|
//export goZapLogExt
|
|
func goZapLogExt(sev C.int,
|
|
file *C.char,
|
|
line C.int,
|
|
msg *C.char,
|
|
msgLen C.int,
|
|
) {
|
|
lv := mapGlogSeverity(int(sev))
|
|
if !log.L().Core().Enabled(lv) {
|
|
return
|
|
}
|
|
ent := zapcore.Entry{
|
|
Level: mapGlogSeverity(int(sev)),
|
|
Time: time.Now(),
|
|
LoggerName: cgoLoggerName,
|
|
Message: C.GoStringN(msg, msgLen),
|
|
Caller: zapcore.EntryCaller{
|
|
Defined: true,
|
|
File: C.GoString(file),
|
|
Line: int(line),
|
|
},
|
|
}
|
|
if ce := log.L().Core().Check(ent, nil); ce != nil {
|
|
metrics.LoggingCGOWriteTotal.Inc()
|
|
metrics.LoggingCGOWriteBytes.Add(float64(msgLen))
|
|
ce.Write()
|
|
}
|
|
}
|
|
|
|
func mapGlogSeverity(s int) zapcore.Level {
|
|
switch s {
|
|
case 0: // GLOG_INFO
|
|
return zapcore.InfoLevel
|
|
case 1: // GLOG_WARNING
|
|
return zapcore.WarnLevel
|
|
case 2: // GLOG_ERROR
|
|
return zapcore.ErrorLevel
|
|
case 3: // GLOG_FATAL
|
|
// glog fatal will call std::abort,
|
|
// zap will call os.Exit(1),
|
|
// we don't want to double exit, so we use error level instead
|
|
return zapcore.ErrorLevel
|
|
default:
|
|
return zapcore.InfoLevel
|
|
}
|
|
}
|