[Fix] [CPU] memory: Honor cgroup memory.max when sizing CPU memory (#40066)

Co-authored-by: Zhang, Mingxu <mingxu.zhang@intel.com>
Co-authored-by: MingxuZh <109504044+MingxuZh@users.noreply.github.com>
This commit is contained in:
Zaili Wang
2026-10-19 11:11:44 +08:00
committed by GitHub
co-authored by Zhang, Mingxu MingxuZh
parent 617c8b67b1
commit f02458fff1
3 changed files with 115 additions and 76 deletions
@@ -22,7 +22,6 @@ import logging
from contextlib import contextmanager
from typing import TYPE_CHECKING, Callable, Optional, Union
import psutil
import torch
import tqdm
@@ -48,6 +47,7 @@ from sglang.srt.runtime_context import (
)
from sglang.srt.utils import (
empty_context,
get_free_cpu_memory,
log_info_on_rank0,
require_attn_tp_gather,
require_gathered_buffer,
@@ -741,7 +741,7 @@ class CPUGraphRunner:
)
for bs in capture_range:
if get_parallel().tp_rank == 0:
avail_mem = psutil.virtual_memory().available / (1 << 30)
avail_mem = get_free_cpu_memory() / (1 << 30)
capture_range.set_description(
f"Capturing batches ({bs=} {avail_mem=:.2f} GB)"
)
+47 -31
View File
@@ -495,7 +495,7 @@ def get_available_gpu_memory(
elif device == "cpu":
# TODO: rename the variables in the current function to be not GPU specific
total_free_memory = psutil.virtual_memory().available
total_free_memory = get_free_cpu_memory()
n_numa_node: int = len(get_cpu_ids_by_node())
free_gpu_memory = round(total_free_memory / n_numa_node, 3)
elif device == "npu":
@@ -880,35 +880,51 @@ def get_npu_memory_capacity():
raise ImportError("torch_npu is required when run on npu device.")
def get_instance_memory_status(cgroup_file: str) -> Optional[int]:
try:
with open(cgroup_file, "r") as f:
content = f.read().strip()
# "max" or other non memory size content will be skipped
if not content.isdigit():
raise ValueError
return int(content)
except (PermissionError, FileNotFoundError, ValueError):
return None
def get_free_cpu_memory() -> int:
free_cpu_memory = psutil.virtual_memory().available
instance_max_cpu_memory = get_instance_memory_status("/sys/fs/cgroup/memory.max")
instance_current_cpu_memory = get_instance_memory_status(
"/sys/fs/cgroup/memory.current"
)
if instance_max_cpu_memory is not None and instance_current_cpu_memory is not None:
instance_free_cpu_memory = max(
0, instance_max_cpu_memory - instance_current_cpu_memory
)
free_cpu_memory = min(free_cpu_memory, instance_free_cpu_memory)
return free_cpu_memory
def get_cpu_memory_capacity():
# Per-rank memory capacity cannot be determined for customized core settings
if os.environ.get("SGLANG_CPU_OMP_THREADS_BIND", ""):
logger.warning(
"SGLANG_CPU_OMP_THREADS_BIND is set for explicit control of CPU cores for each TP rank."
"Please beware that in this case the available memory amounts of the ranks cannot be determined in prior."
"You need to set proper --max-total-tokens to avoid the out-of-memory error."
)
return None
n_numa_node: int = len(get_cpu_ids_by_node())
if n_numa_node == 0:
# Cannot determine NUMA config, fallback to total memory and avoid ZeroDivisionError.
return float(psutil.virtual_memory().total // (1 << 20))
try:
numa_mem_list = list()
file_prefix = "/sys/devices/system/node/"
for numa_id in range(n_numa_node):
file_meminfo = f"node{numa_id}/meminfo"
with open(os.path.join(file_prefix, file_meminfo), "r") as f:
# MemTotal info is at the 1st line
line = f.readline()
# Expected format: "Node 0 MemTotal: 100000000 kB"
parts = line.split()
if len(parts) >= 4 and parts[2] == "MemTotal:":
numa_mem_list.append(int(parts[3]))
else:
raise ValueError(f"Unexpected format in {file_meminfo}: {line}")
# Retrieved value in KB, need MB
numa_mem = float(min(numa_mem_list) // 1024)
return numa_mem
except (FileNotFoundError, ValueError, IndexError):
numa_mem = psutil.virtual_memory().total / n_numa_node
# Retrieved value in Byte, need MB
return float(numa_mem // (1 << 20))
n_numa_node = len(get_cpu_ids_by_node()) or 1
total_cpu_memory = psutil.virtual_memory().total
instance_max_cpu_memory = get_instance_memory_status("/sys/fs/cgroup/memory.max")
if instance_max_cpu_memory is not None:
total_cpu_memory = min(total_cpu_memory, instance_max_cpu_memory)
per_numa_mem = total_cpu_memory / n_numa_node
# Retrieved value in Byte, need MB
return float(per_numa_mem // (1 << 20))
def get_xpu_memory_capacity():
@@ -4594,12 +4610,12 @@ def get_physical_cpus_by_numa():
def get_cpu_ids_by_node():
node_to_cpus = get_physical_cpus_by_numa()
# Sort by NUMA node index
cpu_ids = [
",".join(map(str, sorted(node_to_cpus[node]))) for node in sorted(node_to_cpus)
]
# ['0,1,2,3', '4,5,6,7', '8,9,10,11', '12,13,14,15', '16,17,18,19', '20,21,22,23']
return cpu_ids
return [
",".join(map(str, sorted(cpus)))
for _, cpus in sorted(node_to_cpus.items())
if cpus
]
def is_shm_available(dtype, world_size, local_size):