Files
sglang/python/sglang/srt/mem_cache/host_memory.py
T

138 lines
5.0 KiB
Python

"""Host-memory headroom bounded by the process's visible cgroup hierarchy."""
from __future__ import annotations
import functools
import logging
import re
from pathlib import Path, PurePosixPath
import psutil
from sglang.srt.environ import envs
logger = logging.getLogger(__name__)
_BUDGET_ENV = "SGLANG_HICACHE_HOST_MEMORY_BYTES"
@functools.cache
def _warn_no_cgroup_mount() -> None:
logger.warning(
"The process belongs to a cgroup but no cgroup filesystem is mounted; "
"sizing HiCache against host available memory. If the process runs "
"under a memory limit, set %s to its host memory budget.",
_BUDGET_ENV,
)
def _unescape_mount_path(value: str) -> str:
return re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)
def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
memberships = {}
try:
cgroups = (proc_root / "self/cgroup").read_text()
mounts = (proc_root / "self/mountinfo").read_text()
except FileNotFoundError:
# Non-Linux systems need not expose procfs.
return None
for line in cgroups.splitlines():
_, controllers, path = line.split(":", 2)
if not controllers:
memberships["cgroup2"] = PurePosixPath(path)
elif "memory" in controllers.split(","):
memberships["cgroup"] = PurePosixPath(path)
headroom = None
resolved = False
cgroup_mounted = False
for line in mounts.splitlines():
before, after = line.split(" - ", 1)
filesystem, _, options = after.split()[:3]
cgroup_mounted |= filesystem in ("cgroup", "cgroup2")
if filesystem not in memberships:
continue
if filesystem == "cgroup" and "memory" not in options.split(","):
continue
fields = before.split()
root = PurePosixPath(_unescape_mount_path(fields[3]))
mount = Path(_unescape_mount_path(fields[4]))
membership = memberships[filesystem]
if membership.is_relative_to(root):
relative = membership.relative_to(root)
elif root != PurePosixPath("/"):
# A cgroup namespace can expose membership relative to its root,
# while mountinfo still identifies the host-side subtree.
relative = membership.relative_to("/")
else:
continue
if ".." in relative.parts:
raise ValueError(f"Cannot resolve cgroup memory path: {membership}")
directory = mount / relative
if not directory.is_dir():
continue
resolved = True
limits = (
("memory.max", "memory.high")
if filesystem == "cgroup2"
else ("memory.limit_in_bytes",)
)
usage_name = (
"memory.current" if filesystem == "cgroup2" else "memory.usage_in_bytes"
)
while True:
for name in limits:
try:
value = (directory / name).read_text().strip()
except FileNotFoundError:
# The hierarchy root may not have memory controller files.
continue
if value == "max":
continue
limit = int(value)
# Do not silently ignore an unreadable usage file for a known
# limit: falling back to host RAM could overrun the container.
usage = int((directory / usage_name).read_text())
remaining = max(0, limit - usage)
headroom = remaining if headroom is None else min(headroom, remaining)
if directory == mount:
break
directory = directory.parent
if memberships and not resolved:
if not cgroup_mounted:
# E.g. a container sharing the host cgroup namespace without
# cgroupfs: no limit is visible, so size against host memory.
_warn_no_cgroup_mount()
return None
# A mounted but unmatched hierarchy may hide a real limit.
raise RuntimeError(
"Cannot locate the process memory cgroup in mounted cgroup "
f"filesystems; set {_BUDGET_ENV} to the host memory budget to skip "
"cgroup discovery"
)
return headroom
def available_host_memory_bytes() -> int:
"""Conservative allocatable RAM; charged file cache is not assumed reclaimable.
An explicit SGLANG_HICACHE_HOST_MEMORY_BYTES replaces both probes.
"""
budget = envs.SGLANG_HICACHE_HOST_MEMORY_BYTES.get()
if budget is not None:
if budget <= 0:
raise ValueError(f"{_BUDGET_ENV} must be positive, got {budget}")
return budget
available = psutil.virtual_memory().available
cgroup_headroom = _cgroup_memory_headroom()
if cgroup_headroom is not None:
logger.info(
"HiCache memory headroom: host %.1f GiB, cgroup %.1f GiB",
available / 1024**3,
cgroup_headroom / 1024**3,
)
available = min(available, cgroup_headroom)
return available