mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-01-07 19:31:51 +08:00
27525d57cc
issue: #45640 - After async logging, the C log and go log has no order promise, meanwhile the C log format is not consistent with Go Log; so we close the output of glog, just forward the log result operation into Go side which will be handled by the async zap logger. - Use CGO to filter all cgo logging and promise the order between c log and go log. - Also fix the metric name, add new metric to count the logging. - TODO: after woodpecker use the logger of milvus, we can add bigger buffer for logging. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> - Core invariant: all C (glog) and Go logs must be routed through the same zap async pipeline so ordering and formatting are preserved; this PR ensures every glog emission is captured and forwarded to zap before any async buffering diverges the outputs. - Logic removed/simplified: direct glog outputs and hard stdout/stderr/log_dir settings are disabled (configs/glog.conf and flags in internal/core/src/config/ConfigKnowhere.cpp) because they are redundant once a single zap sink handles all logs; logging metrics were simplified from per-length/volatile gauges to totalized counters (pkg/metrics/logging_metrics.go & pkg/log/*), removing duplicate length-tracking and making accounting consistent. - No data loss or behavior regression (concrete code paths): Google logging now adds a GoZapSink (internal/core/src/common/logging_c.h, logging_c.cpp) that calls the exported CGO bridge goZapLogExt (internal/util/cgo/logging/logging.go). Go side uses C.GoStringN/C.GoString to capture full message and file, maps glog severities to zapcore levels, preserves caller info, and writes via the existing zap async core (same write path used by Go logs). The C++ send() trims glog's trailing newline and forwards exact buffers/lengths, so message content, file, line, and severity are preserved and serialized through the same async writer—no log entries are dropped or reordered relative to Go logs. - Capability added (where it takes effect): a CGO bridge that forwards glog into zap—new Go-exported function goZapLogExt (internal/util/cgo/logging/logging.go), a GoZapSink in C++ that forwards glog sends (internal/core/src/common/logging_c.h/.cpp), and blank imports of the cgo initializer across multiple packages (various internal/* files) to ensure the bridge is registered early so all C logs are captured. <!-- end of auto-generated comment: release notes by coderabbit.ai --> Signed-off-by: chyezh <chyezh@outlook.com>
100 lines
2.2 KiB
Go
100 lines
2.2 KiB
Go
package cgoconverter
|
|
|
|
/*
|
|
#include <stdlib.h>
|
|
*/
|
|
import "C"
|
|
|
|
import (
|
|
"math"
|
|
"sync/atomic"
|
|
"unsafe"
|
|
|
|
_ "github.com/milvus-io/milvus/internal/util/cgo"
|
|
"github.com/milvus-io/milvus/pkg/v2/util/typeutil"
|
|
)
|
|
|
|
const maxByteArrayLen = math.MaxInt32
|
|
|
|
var globalConverter = NewBytesConverter()
|
|
|
|
type BytesConverter struct {
|
|
pointers *typeutil.ConcurrentMap[int32, unsafe.Pointer] // leaseId -> unsafe.Pointer
|
|
nextLease int32
|
|
}
|
|
|
|
func NewBytesConverter() *BytesConverter {
|
|
return &BytesConverter{
|
|
pointers: typeutil.NewConcurrentMap[int32, unsafe.Pointer](),
|
|
nextLease: 0,
|
|
}
|
|
}
|
|
|
|
func (converter *BytesConverter) add(p unsafe.Pointer) int32 {
|
|
lease := atomic.AddInt32(&converter.nextLease, 1)
|
|
converter.pointers.Insert(lease, p)
|
|
|
|
return lease
|
|
}
|
|
|
|
// Return a lease and []byte from C bytes (typically char*)
|
|
// which references the same memory of C bytes
|
|
// Call Release(lease) after you don't need the returned []byte
|
|
func (converter *BytesConverter) UnsafeGoBytes(cbytes *unsafe.Pointer, len int) (int32, []byte) {
|
|
var (
|
|
goBytes []byte
|
|
lease int32
|
|
)
|
|
|
|
if len > maxByteArrayLen {
|
|
// C.GoBytes takes the length as C.int,
|
|
// which is always 32-bit (not depends on platform)
|
|
panic("UnsafeGoBytes: out of length")
|
|
}
|
|
goBytes = (*[maxByteArrayLen]byte)(*cbytes)[:len:len]
|
|
lease = converter.add(*cbytes)
|
|
*cbytes = nil
|
|
|
|
return lease, goBytes
|
|
}
|
|
|
|
func (converter *BytesConverter) Release(lease int32) {
|
|
p := converter.Extract(lease)
|
|
|
|
C.free(p)
|
|
}
|
|
|
|
func (converter *BytesConverter) Extract(lease int32) unsafe.Pointer {
|
|
p, ok := converter.pointers.GetAndRemove(lease)
|
|
if !ok {
|
|
panic("try to release the resource that doesn't exist")
|
|
}
|
|
|
|
return p
|
|
}
|
|
|
|
// Make sure only the caller own the converter
|
|
// or this would release someone's memory
|
|
func (converter *BytesConverter) ReleaseAll() {
|
|
converter.pointers.Range(func(lease int32, pointer unsafe.Pointer) bool {
|
|
converter.pointers.GetAndRemove(lease)
|
|
C.free(pointer)
|
|
|
|
return true
|
|
})
|
|
}
|
|
|
|
func UnsafeGoBytes(cbytes *unsafe.Pointer, len int) (int32, []byte) {
|
|
return globalConverter.UnsafeGoBytes(cbytes, len)
|
|
}
|
|
|
|
func Release(lease int32) {
|
|
globalConverter.Release(lease)
|
|
}
|
|
|
|
func Extract(lease int32) unsafe.Pointer {
|
|
return globalConverter.Extract(lease)
|
|
}
|
|
|
|
// DO NOT provide ReleaseAll() method for global converter
|