admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-28 14:35:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: change jemalloc monitor from compile-time to runtime (#46377)

Browse Source 
#46133

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>
This commit is contained in:
zhagnlu 2025-12-17 16:17:16 +08:00 committed by GitHub
parent 21ed1fabfd
commit 52026cf07e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 31 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
internal/core/CMakeLists.txt
View File

@ -37,11 +37,6 @@ if (USE_OPENDAL)
add_definitions(-DUSE_OPENDAL)
endif()
# Enable jemalloc memory statistics on Linux only
if (LINUX)
add_definitions(-DMILVUS_JEMALLOC_ENABLED)
endif()
project(core)
include(CheckCXXCompilerFlag)
if ( APPLE )

5
internal/core/src/CMakeLists.txt
View File

@ -96,11 +96,6 @@ if (ENABLE_GCP_NATIVE)
set(LINK_TARGETS ${LINK_TARGETS} gcp-native-storage)
endif()
# Link jemalloc library for mallctl API access (Linux only)
if (LINUX)
set(LINK_TARGETS ${LINK_TARGETS} jemalloc)
endif()
target_link_libraries(milvus_core ${LINK_TARGETS})
install(TARGETS milvus_core DESTINATION "${CMAKE_INSTALL_LIBDIR}")

6
internal/core/src/monitor/CMakeLists.txt
View File

@ -11,9 +11,3 @@
add_source_at_current_directory_recursively()
add_library(milvus_monitor OBJECT ${SOURCE_FILES})
# jemalloc_stats_c.cpp needs jemalloc headers on Linux, ensure jemalloc is built first
if (LINUX)
add_dependencies(milvus_monitor jemalloc)
target_include_directories(milvus_monitor PRIVATE ${CMAKE_INSTALL_PREFIX}/include)
endif()

57
internal/core/src/monitor/jemalloc_stats_c.cpp
View File

@ -14,35 +14,44 @@
#include "monitor/jemalloc_stats_c.h"
#ifdef MILVUS_JEMALLOC_ENABLED
#include <jemalloc/jemalloc.h>
#include <cstddef>
#include <cstring>
#ifdef __linux__
#include <dlfcn.h>
#endif
// jemalloc mallctl function type
typedef int (*mallctl_t)(const char*, void*, size_t*, void*, size_t);
// get mallctl function pointer at runtime via dlsym
// Returns nullptr if jemalloc is not loaded (e.g., via LD_PRELOAD)
static mallctl_t
get_mallctl() {
#ifdef __linux__
static mallctl_t fn = (mallctl_t)dlsym(RTLD_DEFAULT, "mallctl");
return fn;
#else
return nullptr;
#endif
}
JemallocStats
GetJemallocStats() {
JemallocStats stats;
// initialize all fields to zero
#ifdef MILVUS_JEMALLOC_ENABLED
std::memset(&stats, 0, sizeof(JemallocStats));
#else
stats.allocated = 0;
stats.active = 0;
stats.metadata = 0;
stats.resident = 0;
stats.mapped = 0;
stats.retained = 0;
stats.fragmentation = 0;
stats.overhead = 0;
#endif
stats.success = false;
#ifdef MILVUS_JEMALLOC_ENABLED
mallctl_t mallctl_fn = get_mallctl();
if (mallctl_fn == nullptr) {
// jemalloc is not available, return zeros
return stats;
}
// refresh jemalloc stats epoch to get current values
uint64_t epoch = 1;
size_t epoch_sz = sizeof(epoch);
if (mallctl("epoch", &epoch, &epoch_sz, &epoch, epoch_sz) != 0) {
if (mallctl_fn("epoch", &epoch, &epoch_sz, &epoch, epoch_sz) != 0) {
return stats;
}
@ -55,32 +64,32 @@ GetJemallocStats() {
size_t retained = 0;
// get allocated bytes (total allocated by application)
if (mallctl("stats.allocated", &allocated, &sz, nullptr, 0) != 0) {
if (mallctl_fn("stats.allocated", &allocated, &sz, nullptr, 0) != 0) {
return stats;
}
// get active bytes (in active pages, includes fragmentation)
if (mallctl("stats.active", &active, &sz, nullptr, 0) != 0) {
if (mallctl_fn("stats.active", &active, &sz, nullptr, 0) != 0) {
return stats;
}
// get metadata bytes (jemalloc's own overhead)
if (mallctl("stats.metadata", &metadata, &sz, nullptr, 0) != 0) {
if (mallctl_fn("stats.metadata", &metadata, &sz, nullptr, 0) != 0) {
return stats;
}
// get resident bytes (physically resident memory, RSS)
if (mallctl("stats.resident", &resident, &sz, nullptr, 0) != 0) {
if (mallctl_fn("stats.resident", &resident, &sz, nullptr, 0) != 0) {
return stats;
}
// get mapped bytes (total virtual memory mappings)
if (mallctl("stats.mapped", &mapped, &sz, nullptr, 0) != 0) {
if (mallctl_fn("stats.mapped", &mapped, &sz, nullptr, 0) != 0) {
return stats;
}
// get retained bytes (virtual memory that could be returned to OS)
if (mallctl("stats.retained", &retained, &sz, nullptr, 0) != 0) {
if (mallctl_fn("stats.retained", &retained, &sz, nullptr, 0) != 0) {
return stats;
}
@ -99,10 +108,6 @@ GetJemallocStats() {
stats.overhead = (resident > active) ? (resident - active) : 0;
stats.success = true;
#else
// on platforms without jemalloc (e.g., macOS), return empty stats
// with success = false to indicate jemalloc is not available
#endif
return stats;
}