fix: added more comprehensive container limit detection (#43693)

issue: #41435

Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>
This commit is contained in:
Buqian Zheng 2025-08-01 20:37:37 +08:00 committed by GitHub
parent ecc2ac0426
commit b0226ef47c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 70 additions and 24 deletions

View File

@ -92,7 +92,7 @@ getContainerMemLimit() {
try { try {
int64_t env_limit = std::stoll(mem_limit_env); int64_t env_limit = std::stoll(mem_limit_env);
limits.push_back(env_limit); limits.push_back(env_limit);
LOG_DEBUG("[MCL] Found MEM_LIMIT environment variable: {}", LOG_TRACE("[MCL] Found MEM_LIMIT environment variable: {}",
FormatBytes(env_limit)); FormatBytes(env_limit));
} catch (...) { } catch (...) {
LOG_WARN("[MCL] Invalid MEM_LIMIT environment variable: {}", LOG_WARN("[MCL] Invalid MEM_LIMIT environment variable: {}",
@ -100,6 +100,46 @@ getContainerMemLimit() {
} }
} }
// Check direct cgroup v1 path
std::ifstream cgroup_v1("/sys/fs/cgroup/memory/memory.limit_in_bytes");
if (cgroup_v1.is_open()) {
std::string limit_str;
if (std::getline(cgroup_v1, limit_str)) {
try {
int64_t limit = std::stoll(limit_str);
if (limit < (1LL << 62)) { // Filter out unrealistic values
limits.push_back(limit);
LOG_TRACE(
"[MCL] Found direct cgroups v1 limit "
"(/sys/fs/cgroup/memory/memory.limit_in_bytes): {}",
FormatBytes(limit));
}
} catch (...) {
// Ignore parse errors
}
}
cgroup_v1.close();
}
// Check direct cgroup v2 path
std::ifstream cgroup_v2("/sys/fs/cgroup/memory.max");
if (cgroup_v2.is_open()) {
std::string limit_str;
if (std::getline(cgroup_v2, limit_str) && limit_str != "max") {
try {
int64_t limit = std::stoll(limit_str);
limits.push_back(limit);
LOG_TRACE(
"[MCL] Found direct cgroups v2 limit "
"(/sys/fs/cgroup/memory.max): {}",
FormatBytes(limit));
} catch (...) {
// Ignore parse errors
}
}
cgroup_v2.close();
}
// Check process-specific cgroup limits from /proc/self/cgroup // Check process-specific cgroup limits from /proc/self/cgroup
std::ifstream proc_cgroup("/proc/self/cgroup"); std::ifstream proc_cgroup("/proc/self/cgroup");
if (proc_cgroup.is_open()) { if (proc_cgroup.is_open()) {
@ -123,7 +163,7 @@ getContainerMemLimit() {
try { try {
int64_t proc_limit = std::stoll(proc_line); int64_t proc_limit = std::stoll(proc_line);
limits.push_back(proc_limit); limits.push_back(proc_limit);
LOG_DEBUG( LOG_TRACE(
"[MCL] Found process-specific cgroups v2 " "[MCL] Found process-specific cgroups v2 "
"limit: {}", "limit: {}",
FormatBytes(proc_limit)); FormatBytes(proc_limit));
@ -147,7 +187,7 @@ getContainerMemLimit() {
// when unlimited) // when unlimited)
if (proc_limit < (1LL << 62)) { if (proc_limit < (1LL << 62)) {
limits.push_back(proc_limit); limits.push_back(proc_limit);
LOG_DEBUG( LOG_TRACE(
"[MCL] Found process-specific cgroups " "[MCL] Found process-specific cgroups "
"v1 limit: {}", "v1 limit: {}",
FormatBytes(proc_limit)); FormatBytes(proc_limit));
@ -166,7 +206,7 @@ getContainerMemLimit() {
// Return the minimum of all found limits // Return the minimum of all found limits
if (!limits.empty()) { if (!limits.empty()) {
int64_t min_limit = *std::min_element(limits.begin(), limits.end()); int64_t min_limit = *std::min_element(limits.begin(), limits.end());
LOG_DEBUG("[MCL] Using minimum memory limit: {} from {} sources", LOG_TRACE("[MCL] Using minimum memory limit: {} from {} sources",
FormatBytes(min_limit), FormatBytes(min_limit),
limits.size()); limits.size());
return min_limit; return min_limit;

View File

@ -200,29 +200,35 @@ std::string
DList::usageInfo() const { DList::usageInfo() const {
auto used = used_resources_.load(); auto used = used_resources_.load();
static double precision = 100.0; static double precision = 100.0;
return fmt::format( std::string info = fmt::format(
"low_watermark_: {}; " "low_watermark_: {}; high_watermark_: {}; "
"high_watermark_: {}; " "max_memory_: {}; used_resources_: {} (",
"max_memory_: {}; "
"used_resources_: {} {:.2}% of max, {:.2}% of "
"high_watermark memory, {:.2}% of max, {:.2}% of "
"high_watermark disk; "
"evictable_size_: {}; "
"loading: {}; ",
low_watermark_.ToString(), low_watermark_.ToString(),
high_watermark_.ToString(), high_watermark_.ToString(),
max_memory_.ToString(), max_memory_.ToString(),
used.ToString(), used.ToString());
static_cast<double>(used.memory_bytes) / max_memory_.memory_bytes *
precision, if (used.memory_bytes > 0) {
static_cast<double>(used.memory_bytes) / high_watermark_.memory_bytes * info += fmt::format(", {:.2}% of max, {:.2}% of high_watermark memory",
precision, static_cast<double>(used.memory_bytes) /
static_cast<double>(used.file_bytes) / max_memory_.file_bytes * max_memory_.memory_bytes * precision,
precision, static_cast<double>(used.memory_bytes) /
static_cast<double>(used.file_bytes) / high_watermark_.file_bytes * high_watermark_.memory_bytes * precision);
precision, }
if (used.file_bytes > 0) {
info += fmt::format(", {:.2}% of max, {:.2}% of high_watermark disk",
static_cast<double>(used.file_bytes) /
max_memory_.file_bytes * precision,
static_cast<double>(used.file_bytes) /
high_watermark_.file_bytes * precision);
}
info += fmt::format("); evictable_size_: {}; loading: {}; ",
evictable_size_.load().ToString(), evictable_size_.load().ToString(),
loading_.load().ToString()); loading_.load().ToString());
return info;
} }
// this method is not thread safe, it does not attempt to lock each node, use for debug only. // this method is not thread safe, it does not attempt to lock each node, use for debug only.