[Fix] [CPU] memory: Honor cgroup memory.max when sizing CPU memory (#40066)
Co-authored-by: Zhang, Mingxu <mingxu.zhang@intel.com> Co-authored-by: MingxuZh <109504044+MingxuZh@users.noreply.github.com>
This commit is contained in:
co-authored by
Zhang, Mingxu
MingxuZh
parent
617c8b67b1
commit
f02458fff1
@@ -22,7 +22,6 @@ import logging
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING, Callable, Optional, Union
|
||||
|
||||
import psutil
|
||||
import torch
|
||||
import tqdm
|
||||
|
||||
@@ -48,6 +47,7 @@ from sglang.srt.runtime_context import (
|
||||
)
|
||||
from sglang.srt.utils import (
|
||||
empty_context,
|
||||
get_free_cpu_memory,
|
||||
log_info_on_rank0,
|
||||
require_attn_tp_gather,
|
||||
require_gathered_buffer,
|
||||
@@ -741,7 +741,7 @@ class CPUGraphRunner:
|
||||
)
|
||||
for bs in capture_range:
|
||||
if get_parallel().tp_rank == 0:
|
||||
avail_mem = psutil.virtual_memory().available / (1 << 30)
|
||||
avail_mem = get_free_cpu_memory() / (1 << 30)
|
||||
capture_range.set_description(
|
||||
f"Capturing batches ({bs=} {avail_mem=:.2f} GB)"
|
||||
)
|
||||
|
||||
@@ -495,7 +495,7 @@ def get_available_gpu_memory(
|
||||
|
||||
elif device == "cpu":
|
||||
# TODO: rename the variables in the current function to be not GPU specific
|
||||
total_free_memory = psutil.virtual_memory().available
|
||||
total_free_memory = get_free_cpu_memory()
|
||||
n_numa_node: int = len(get_cpu_ids_by_node())
|
||||
free_gpu_memory = round(total_free_memory / n_numa_node, 3)
|
||||
elif device == "npu":
|
||||
@@ -880,35 +880,51 @@ def get_npu_memory_capacity():
|
||||
raise ImportError("torch_npu is required when run on npu device.")
|
||||
|
||||
|
||||
def get_instance_memory_status(cgroup_file: str) -> Optional[int]:
|
||||
try:
|
||||
with open(cgroup_file, "r") as f:
|
||||
content = f.read().strip()
|
||||
# "max" or other non memory size content will be skipped
|
||||
if not content.isdigit():
|
||||
raise ValueError
|
||||
|
||||
return int(content)
|
||||
except (PermissionError, FileNotFoundError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def get_free_cpu_memory() -> int:
|
||||
free_cpu_memory = psutil.virtual_memory().available
|
||||
instance_max_cpu_memory = get_instance_memory_status("/sys/fs/cgroup/memory.max")
|
||||
instance_current_cpu_memory = get_instance_memory_status(
|
||||
"/sys/fs/cgroup/memory.current"
|
||||
)
|
||||
if instance_max_cpu_memory is not None and instance_current_cpu_memory is not None:
|
||||
instance_free_cpu_memory = max(
|
||||
0, instance_max_cpu_memory - instance_current_cpu_memory
|
||||
)
|
||||
free_cpu_memory = min(free_cpu_memory, instance_free_cpu_memory)
|
||||
|
||||
return free_cpu_memory
|
||||
|
||||
|
||||
def get_cpu_memory_capacity():
|
||||
# Per-rank memory capacity cannot be determined for customized core settings
|
||||
if os.environ.get("SGLANG_CPU_OMP_THREADS_BIND", ""):
|
||||
logger.warning(
|
||||
"SGLANG_CPU_OMP_THREADS_BIND is set for explicit control of CPU cores for each TP rank."
|
||||
"Please beware that in this case the available memory amounts of the ranks cannot be determined in prior."
|
||||
"You need to set proper --max-total-tokens to avoid the out-of-memory error."
|
||||
)
|
||||
return None
|
||||
n_numa_node: int = len(get_cpu_ids_by_node())
|
||||
if n_numa_node == 0:
|
||||
# Cannot determine NUMA config, fallback to total memory and avoid ZeroDivisionError.
|
||||
return float(psutil.virtual_memory().total // (1 << 20))
|
||||
try:
|
||||
numa_mem_list = list()
|
||||
file_prefix = "/sys/devices/system/node/"
|
||||
for numa_id in range(n_numa_node):
|
||||
file_meminfo = f"node{numa_id}/meminfo"
|
||||
with open(os.path.join(file_prefix, file_meminfo), "r") as f:
|
||||
# MemTotal info is at the 1st line
|
||||
line = f.readline()
|
||||
# Expected format: "Node 0 MemTotal: 100000000 kB"
|
||||
parts = line.split()
|
||||
if len(parts) >= 4 and parts[2] == "MemTotal:":
|
||||
numa_mem_list.append(int(parts[3]))
|
||||
else:
|
||||
raise ValueError(f"Unexpected format in {file_meminfo}: {line}")
|
||||
# Retrieved value in KB, need MB
|
||||
numa_mem = float(min(numa_mem_list) // 1024)
|
||||
return numa_mem
|
||||
except (FileNotFoundError, ValueError, IndexError):
|
||||
numa_mem = psutil.virtual_memory().total / n_numa_node
|
||||
# Retrieved value in Byte, need MB
|
||||
return float(numa_mem // (1 << 20))
|
||||
n_numa_node = len(get_cpu_ids_by_node()) or 1
|
||||
total_cpu_memory = psutil.virtual_memory().total
|
||||
instance_max_cpu_memory = get_instance_memory_status("/sys/fs/cgroup/memory.max")
|
||||
if instance_max_cpu_memory is not None:
|
||||
total_cpu_memory = min(total_cpu_memory, instance_max_cpu_memory)
|
||||
per_numa_mem = total_cpu_memory / n_numa_node
|
||||
# Retrieved value in Byte, need MB
|
||||
return float(per_numa_mem // (1 << 20))
|
||||
|
||||
|
||||
def get_xpu_memory_capacity():
|
||||
@@ -4594,12 +4610,12 @@ def get_physical_cpus_by_numa():
|
||||
def get_cpu_ids_by_node():
|
||||
node_to_cpus = get_physical_cpus_by_numa()
|
||||
# Sort by NUMA node index
|
||||
cpu_ids = [
|
||||
",".join(map(str, sorted(node_to_cpus[node]))) for node in sorted(node_to_cpus)
|
||||
]
|
||||
|
||||
# ['0,1,2,3', '4,5,6,7', '8,9,10,11', '12,13,14,15', '16,17,18,19', '20,21,22,23']
|
||||
return cpu_ids
|
||||
return [
|
||||
",".join(map(str, sorted(cpus)))
|
||||
for _, cpus in sorted(node_to_cpus.items())
|
||||
if cpus
|
||||
]
|
||||
|
||||
|
||||
def is_shm_available(dtype, world_size, local_size):
|
||||
|
||||
Reference in New Issue
Block a user