[HiCache] Fix cgroup page-cache accounting and the sizing fallback when cgroup discovery fails (#42420)

Co-authored-by: merrymercy <merrymercy@users.noreply.github.com>
Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
Co-authored-by: nvjullin <nvjullin@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-03 16:01:32 -07:00
committed by GitHub
co-authored by merrymercy xiezhq-hermann nvjullin
parent 4301be7bdb
commit 4ab720e655
5 changed files with 113 additions and 78 deletions
-4
View File
@@ -765,10 +765,6 @@ class Envs:
# HiCache host<->device transfers use the TMA staging kernel when the GPU
# (sm_90+), row size and page size allow; set to 0 to force the register kernel.
SGLANG_HICACHE_TMA_TRANSFER = EnvBool(True)
# Host memory in bytes available to HiCache pools on this host. When set,
# it replaces cgroup and psutil discovery; the 10 GiB reserve and the
# per-local-rank split still apply.
SGLANG_HICACHE_HOST_MEMORY_BYTES = EnvInt(None)
# Base token count for each MLA/DSA dedup broadcast chunk.
SGLANG_MLA_DEDUP_CHUNK_TOKENS = EnvInt(2048)
SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None)
@@ -93,7 +93,7 @@ def auto_size_hicache(
return
requested = get_memory().hicache_ratio
device_bytes = _estimate_hicache_bytes(params, draft_plan)
budget = int(host_memory_budget_bytes() * fraction)
budget = int(host_memory_budget_bytes(auto_size=True) * fraction)
ratio = min(requested, budget * (1 - _ALLOCATION_SLACK_FRACTION) / device_bytes)
# One collective before any pool is built: PP stages own different pool
# counts, so a per-pool collective could deadlock.
+35 -38
View File
@@ -2,34 +2,24 @@
from __future__ import annotations
import functools
import logging
import re
from pathlib import Path, PurePosixPath
import psutil
from sglang.srt.environ import envs
logger = logging.getLogger(__name__)
_BUDGET_ENV = "SGLANG_HICACHE_HOST_MEMORY_BYTES"
@functools.cache
def _warn_no_cgroup_mount() -> None:
logger.warning(
"The process belongs to a cgroup but no cgroup filesystem is mounted; "
"sizing HiCache against host available memory. If the process runs "
"under a memory limit, set %s to its host memory budget.",
_BUDGET_ENV,
)
def _unescape_mount_path(value: str) -> str:
return re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)
def _memory_stat(directory: Path) -> dict[str, int]:
lines = (directory / "memory.stat").read_text().splitlines()
return {key: int(value) for key, value in map(str.split, lines)}
def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
memberships = {}
try:
@@ -47,11 +37,9 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
headroom = None
resolved = False
cgroup_mounted = False
for line in mounts.splitlines():
before, after = line.split(" - ", 1)
filesystem, _, options = after.split()[:3]
cgroup_mounted |= filesystem in ("cgroup", "cgroup2")
if filesystem not in memberships:
continue
if filesystem == "cgroup" and "memory" not in options.split(","):
@@ -82,6 +70,13 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
usage_name = (
"memory.current" if filesystem == "cgroup2" else "memory.usage_in_bytes"
)
# File LRU pages, as MemAvailable counts; shmem sits on the anon LRU.
# v1's unprefixed counters omit descendants, unlike its usage.
page_cache_keys = (
("active_file", "inactive_file")
if filesystem == "cgroup2"
else ("total_active_file", "total_inactive_file")
)
while True:
for name in limits:
try:
@@ -95,38 +90,40 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
# Do not silently ignore an unreadable usage file for a known
# limit: falling back to host RAM could overrun the container.
usage = int((directory / usage_name).read_text())
remaining = max(0, limit - usage)
stat = _memory_stat(directory)
page_cache = sum(stat[key] for key in page_cache_keys)
# Charged page cache is reclaimed under the limit, not used.
remaining = max(0, limit - max(0, usage - page_cache))
headroom = remaining if headroom is None else min(headroom, remaining)
if directory == mount:
break
directory = directory.parent
if memberships and not resolved:
if not cgroup_mounted:
# E.g. a container sharing the host cgroup namespace without
# cgroupfs: no limit is visible, so size against host memory.
_warn_no_cgroup_mount()
return None
# A mounted but unmatched hierarchy may hide a real limit.
raise RuntimeError(
"Cannot locate the process memory cgroup in mounted cgroup "
f"filesystems; set {_BUDGET_ENV} to the host memory budget to skip "
"cgroup discovery"
"Cannot locate the process memory cgroup in mounted cgroup filesystems"
)
return headroom
def available_host_memory_bytes() -> int:
"""Conservative allocatable RAM; charged file cache is not assumed reclaimable.
An explicit SGLANG_HICACHE_HOST_MEMORY_BYTES replaces both probes.
"""
budget = envs.SGLANG_HICACHE_HOST_MEMORY_BYTES.get()
if budget is not None:
if budget <= 0:
raise ValueError(f"{_BUDGET_ENV} must be positive, got {budget}")
return budget
def available_host_memory_bytes(*, allow_cgroup_fallback: bool = False) -> int:
"""Allocatable RAM, counting reclaimable page cache as free."""
available = psutil.virtual_memory().available
cgroup_headroom = _cgroup_memory_headroom()
try:
cgroup_headroom = _cgroup_memory_headroom()
except (OSError, RuntimeError) as exc:
if not allow_cgroup_fallback:
raise RuntimeError(
"Cannot automatically size HiCache host memory; "
"set --hicache-size explicitly (GB per rank), leaving room "
"for other memory use within the container limit."
) from exc
logger.warning(
"Cannot determine cgroup memory headroom: %s. Checking explicitly "
"sized HiCache pools against host available memory; ensure all "
"local ranks' pools fit within the container memory limit.",
exc,
)
return available
if cgroup_headroom is not None:
logger.info(
"HiCache memory headroom: host %.1f GiB, cgroup %.1f GiB",
@@ -82,12 +82,16 @@ def ranks_per_host() -> int:
return max(launch_world_size // get_parallel().nnodes, 1)
def host_memory_budget_bytes(requested_bytes: int = 0) -> int:
def host_memory_budget_bytes(
requested_bytes: int = 0, *, auto_size: bool = False
) -> int:
"""Host RAM this rank may claim for a HiCache pool.
Bound machine availability by the visible cgroup limits before splitting
among local ranks. Independent engines with separate container budgets
therefore size against their own remaining allowance.
Auto-sizing requires successful cgroup discovery; explicit pool sizing
can fall back to checking host availability when discovery fails.
Inside host_memory_budget_scope, requested_bytes is booked against the
snapshot when it fits; the allowance before booking is returned.
@@ -98,7 +102,10 @@ def host_memory_budget_bytes(requested_bytes: int = 0) -> int:
_host_memory_budget.set(available - requested_bytes)
return available
free = available_host_memory_bytes() - HICACHE_HOST_MEMORY_RESERVE_BYTES
free = (
available_host_memory_bytes(allow_cgroup_fallback=not auto_size)
- HICACHE_HOST_MEMORY_RESERVE_BYTES
)
return free // ranks_per_host()