mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-08 01:58:34 +08:00
fix: added more comprehensive container limit detection (#43693)
issue: #41435 Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>
This commit is contained in:
parent
ecc2ac0426
commit
b0226ef47c
@ -92,7 +92,7 @@ getContainerMemLimit() {
|
|||||||
try {
|
try {
|
||||||
int64_t env_limit = std::stoll(mem_limit_env);
|
int64_t env_limit = std::stoll(mem_limit_env);
|
||||||
limits.push_back(env_limit);
|
limits.push_back(env_limit);
|
||||||
LOG_DEBUG("[MCL] Found MEM_LIMIT environment variable: {}",
|
LOG_TRACE("[MCL] Found MEM_LIMIT environment variable: {}",
|
||||||
FormatBytes(env_limit));
|
FormatBytes(env_limit));
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
LOG_WARN("[MCL] Invalid MEM_LIMIT environment variable: {}",
|
LOG_WARN("[MCL] Invalid MEM_LIMIT environment variable: {}",
|
||||||
@ -100,6 +100,46 @@ getContainerMemLimit() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check direct cgroup v1 path
|
||||||
|
std::ifstream cgroup_v1("/sys/fs/cgroup/memory/memory.limit_in_bytes");
|
||||||
|
if (cgroup_v1.is_open()) {
|
||||||
|
std::string limit_str;
|
||||||
|
if (std::getline(cgroup_v1, limit_str)) {
|
||||||
|
try {
|
||||||
|
int64_t limit = std::stoll(limit_str);
|
||||||
|
if (limit < (1LL << 62)) { // Filter out unrealistic values
|
||||||
|
limits.push_back(limit);
|
||||||
|
LOG_TRACE(
|
||||||
|
"[MCL] Found direct cgroups v1 limit "
|
||||||
|
"(/sys/fs/cgroup/memory/memory.limit_in_bytes): {}",
|
||||||
|
FormatBytes(limit));
|
||||||
|
}
|
||||||
|
} catch (...) {
|
||||||
|
// Ignore parse errors
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cgroup_v1.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check direct cgroup v2 path
|
||||||
|
std::ifstream cgroup_v2("/sys/fs/cgroup/memory.max");
|
||||||
|
if (cgroup_v2.is_open()) {
|
||||||
|
std::string limit_str;
|
||||||
|
if (std::getline(cgroup_v2, limit_str) && limit_str != "max") {
|
||||||
|
try {
|
||||||
|
int64_t limit = std::stoll(limit_str);
|
||||||
|
limits.push_back(limit);
|
||||||
|
LOG_TRACE(
|
||||||
|
"[MCL] Found direct cgroups v2 limit "
|
||||||
|
"(/sys/fs/cgroup/memory.max): {}",
|
||||||
|
FormatBytes(limit));
|
||||||
|
} catch (...) {
|
||||||
|
// Ignore parse errors
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cgroup_v2.close();
|
||||||
|
}
|
||||||
|
|
||||||
// Check process-specific cgroup limits from /proc/self/cgroup
|
// Check process-specific cgroup limits from /proc/self/cgroup
|
||||||
std::ifstream proc_cgroup("/proc/self/cgroup");
|
std::ifstream proc_cgroup("/proc/self/cgroup");
|
||||||
if (proc_cgroup.is_open()) {
|
if (proc_cgroup.is_open()) {
|
||||||
@ -123,7 +163,7 @@ getContainerMemLimit() {
|
|||||||
try {
|
try {
|
||||||
int64_t proc_limit = std::stoll(proc_line);
|
int64_t proc_limit = std::stoll(proc_line);
|
||||||
limits.push_back(proc_limit);
|
limits.push_back(proc_limit);
|
||||||
LOG_DEBUG(
|
LOG_TRACE(
|
||||||
"[MCL] Found process-specific cgroups v2 "
|
"[MCL] Found process-specific cgroups v2 "
|
||||||
"limit: {}",
|
"limit: {}",
|
||||||
FormatBytes(proc_limit));
|
FormatBytes(proc_limit));
|
||||||
@ -147,7 +187,7 @@ getContainerMemLimit() {
|
|||||||
// when unlimited)
|
// when unlimited)
|
||||||
if (proc_limit < (1LL << 62)) {
|
if (proc_limit < (1LL << 62)) {
|
||||||
limits.push_back(proc_limit);
|
limits.push_back(proc_limit);
|
||||||
LOG_DEBUG(
|
LOG_TRACE(
|
||||||
"[MCL] Found process-specific cgroups "
|
"[MCL] Found process-specific cgroups "
|
||||||
"v1 limit: {}",
|
"v1 limit: {}",
|
||||||
FormatBytes(proc_limit));
|
FormatBytes(proc_limit));
|
||||||
@ -166,7 +206,7 @@ getContainerMemLimit() {
|
|||||||
// Return the minimum of all found limits
|
// Return the minimum of all found limits
|
||||||
if (!limits.empty()) {
|
if (!limits.empty()) {
|
||||||
int64_t min_limit = *std::min_element(limits.begin(), limits.end());
|
int64_t min_limit = *std::min_element(limits.begin(), limits.end());
|
||||||
LOG_DEBUG("[MCL] Using minimum memory limit: {} from {} sources",
|
LOG_TRACE("[MCL] Using minimum memory limit: {} from {} sources",
|
||||||
FormatBytes(min_limit),
|
FormatBytes(min_limit),
|
||||||
limits.size());
|
limits.size());
|
||||||
return min_limit;
|
return min_limit;
|
||||||
|
|||||||
@ -200,29 +200,35 @@ std::string
|
|||||||
DList::usageInfo() const {
|
DList::usageInfo() const {
|
||||||
auto used = used_resources_.load();
|
auto used = used_resources_.load();
|
||||||
static double precision = 100.0;
|
static double precision = 100.0;
|
||||||
return fmt::format(
|
std::string info = fmt::format(
|
||||||
"low_watermark_: {}; "
|
"low_watermark_: {}; high_watermark_: {}; "
|
||||||
"high_watermark_: {}; "
|
"max_memory_: {}; used_resources_: {} (",
|
||||||
"max_memory_: {}; "
|
|
||||||
"used_resources_: {} {:.2}% of max, {:.2}% of "
|
|
||||||
"high_watermark memory, {:.2}% of max, {:.2}% of "
|
|
||||||
"high_watermark disk; "
|
|
||||||
"evictable_size_: {}; "
|
|
||||||
"loading: {}; ",
|
|
||||||
low_watermark_.ToString(),
|
low_watermark_.ToString(),
|
||||||
high_watermark_.ToString(),
|
high_watermark_.ToString(),
|
||||||
max_memory_.ToString(),
|
max_memory_.ToString(),
|
||||||
used.ToString(),
|
used.ToString());
|
||||||
static_cast<double>(used.memory_bytes) / max_memory_.memory_bytes *
|
|
||||||
precision,
|
if (used.memory_bytes > 0) {
|
||||||
static_cast<double>(used.memory_bytes) / high_watermark_.memory_bytes *
|
info += fmt::format(", {:.2}% of max, {:.2}% of high_watermark memory",
|
||||||
precision,
|
static_cast<double>(used.memory_bytes) /
|
||||||
static_cast<double>(used.file_bytes) / max_memory_.file_bytes *
|
max_memory_.memory_bytes * precision,
|
||||||
precision,
|
static_cast<double>(used.memory_bytes) /
|
||||||
static_cast<double>(used.file_bytes) / high_watermark_.file_bytes *
|
high_watermark_.memory_bytes * precision);
|
||||||
precision,
|
}
|
||||||
|
|
||||||
|
if (used.file_bytes > 0) {
|
||||||
|
info += fmt::format(", {:.2}% of max, {:.2}% of high_watermark disk",
|
||||||
|
static_cast<double>(used.file_bytes) /
|
||||||
|
max_memory_.file_bytes * precision,
|
||||||
|
static_cast<double>(used.file_bytes) /
|
||||||
|
high_watermark_.file_bytes * precision);
|
||||||
|
}
|
||||||
|
|
||||||
|
info += fmt::format("); evictable_size_: {}; loading: {}; ",
|
||||||
evictable_size_.load().ToString(),
|
evictable_size_.load().ToString(),
|
||||||
loading_.load().ToString());
|
loading_.load().ToString());
|
||||||
|
|
||||||
|
return info;
|
||||||
}
|
}
|
||||||
|
|
||||||
// this method is not thread safe, it does not attempt to lock each node, use for debug only.
|
// this method is not thread safe, it does not attempt to lock each node, use for debug only.
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user