138 lines
5.0 KiB
Python
138 lines
5.0 KiB
Python
"""Host-memory headroom bounded by the process's visible cgroup hierarchy."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import functools
|
|
import logging
|
|
import re
|
|
from pathlib import Path, PurePosixPath
|
|
|
|
import psutil
|
|
|
|
from sglang.srt.environ import envs
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_BUDGET_ENV = "SGLANG_HICACHE_HOST_MEMORY_BYTES"
|
|
|
|
|
|
@functools.cache
|
|
def _warn_no_cgroup_mount() -> None:
|
|
logger.warning(
|
|
"The process belongs to a cgroup but no cgroup filesystem is mounted; "
|
|
"sizing HiCache against host available memory. If the process runs "
|
|
"under a memory limit, set %s to its host memory budget.",
|
|
_BUDGET_ENV,
|
|
)
|
|
|
|
|
|
def _unescape_mount_path(value: str) -> str:
|
|
return re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)
|
|
|
|
|
|
def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
|
memberships = {}
|
|
try:
|
|
cgroups = (proc_root / "self/cgroup").read_text()
|
|
mounts = (proc_root / "self/mountinfo").read_text()
|
|
except FileNotFoundError:
|
|
# Non-Linux systems need not expose procfs.
|
|
return None
|
|
for line in cgroups.splitlines():
|
|
_, controllers, path = line.split(":", 2)
|
|
if not controllers:
|
|
memberships["cgroup2"] = PurePosixPath(path)
|
|
elif "memory" in controllers.split(","):
|
|
memberships["cgroup"] = PurePosixPath(path)
|
|
|
|
headroom = None
|
|
resolved = False
|
|
cgroup_mounted = False
|
|
for line in mounts.splitlines():
|
|
before, after = line.split(" - ", 1)
|
|
filesystem, _, options = after.split()[:3]
|
|
cgroup_mounted |= filesystem in ("cgroup", "cgroup2")
|
|
if filesystem not in memberships:
|
|
continue
|
|
if filesystem == "cgroup" and "memory" not in options.split(","):
|
|
continue
|
|
fields = before.split()
|
|
root = PurePosixPath(_unescape_mount_path(fields[3]))
|
|
mount = Path(_unescape_mount_path(fields[4]))
|
|
membership = memberships[filesystem]
|
|
if membership.is_relative_to(root):
|
|
relative = membership.relative_to(root)
|
|
elif root != PurePosixPath("/"):
|
|
# A cgroup namespace can expose membership relative to its root,
|
|
# while mountinfo still identifies the host-side subtree.
|
|
relative = membership.relative_to("/")
|
|
else:
|
|
continue
|
|
if ".." in relative.parts:
|
|
raise ValueError(f"Cannot resolve cgroup memory path: {membership}")
|
|
directory = mount / relative
|
|
if not directory.is_dir():
|
|
continue
|
|
resolved = True
|
|
limits = (
|
|
("memory.max", "memory.high")
|
|
if filesystem == "cgroup2"
|
|
else ("memory.limit_in_bytes",)
|
|
)
|
|
usage_name = (
|
|
"memory.current" if filesystem == "cgroup2" else "memory.usage_in_bytes"
|
|
)
|
|
while True:
|
|
for name in limits:
|
|
try:
|
|
value = (directory / name).read_text().strip()
|
|
except FileNotFoundError:
|
|
# The hierarchy root may not have memory controller files.
|
|
continue
|
|
if value == "max":
|
|
continue
|
|
limit = int(value)
|
|
# Do not silently ignore an unreadable usage file for a known
|
|
# limit: falling back to host RAM could overrun the container.
|
|
usage = int((directory / usage_name).read_text())
|
|
remaining = max(0, limit - usage)
|
|
headroom = remaining if headroom is None else min(headroom, remaining)
|
|
if directory == mount:
|
|
break
|
|
directory = directory.parent
|
|
if memberships and not resolved:
|
|
if not cgroup_mounted:
|
|
# E.g. a container sharing the host cgroup namespace without
|
|
# cgroupfs: no limit is visible, so size against host memory.
|
|
_warn_no_cgroup_mount()
|
|
return None
|
|
# A mounted but unmatched hierarchy may hide a real limit.
|
|
raise RuntimeError(
|
|
"Cannot locate the process memory cgroup in mounted cgroup "
|
|
f"filesystems; set {_BUDGET_ENV} to the host memory budget to skip "
|
|
"cgroup discovery"
|
|
)
|
|
return headroom
|
|
|
|
|
|
def available_host_memory_bytes() -> int:
|
|
"""Conservative allocatable RAM; charged file cache is not assumed reclaimable.
|
|
|
|
An explicit SGLANG_HICACHE_HOST_MEMORY_BYTES replaces both probes.
|
|
"""
|
|
budget = envs.SGLANG_HICACHE_HOST_MEMORY_BYTES.get()
|
|
if budget is not None:
|
|
if budget <= 0:
|
|
raise ValueError(f"{_BUDGET_ENV} must be positive, got {budget}")
|
|
return budget
|
|
available = psutil.virtual_memory().available
|
|
cgroup_headroom = _cgroup_memory_headroom()
|
|
if cgroup_headroom is not None:
|
|
logger.info(
|
|
"HiCache memory headroom: host %.1f GiB, cgroup %.1f GiB",
|
|
available / 1024**3,
|
|
cgroup_headroom / 1024**3,
|
|
)
|
|
available = min(available, cgroup_headroom)
|
|
return available
|