From 4ab720e6557b44d07bde471ed52a178795298f1a Mon Sep 17 00:00:00 2001 From: metamergebot Date: Sat, 3 Oct 2026 16:01:32 -0700 Subject: [PATCH] [HiCache] Fix cgroup page-cache accounting and the sizing fallback when cgroup discovery fails (#42420) Co-authored-by: merrymercy Co-authored-by: xiezhq-hermann Co-authored-by: nvjullin --- python/sglang/srt/environ.py | 4 - .../sglang/srt/mem_cache/hicache_auto_size.py | 2 +- python/sglang/srt/mem_cache/host_memory.py | 73 ++++++------- python/sglang/srt/mem_cache/pool_host/base.py | 11 +- .../unit/mem_cache/test_host_memory.py | 101 ++++++++++++------ 5 files changed, 113 insertions(+), 78 deletions(-) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index c01a22f262..031a429099 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -765,10 +765,6 @@ class Envs: # HiCache host<->device transfers use the TMA staging kernel when the GPU # (sm_90+), row size and page size allow; set to 0 to force the register kernel. SGLANG_HICACHE_TMA_TRANSFER = EnvBool(True) - # Host memory in bytes available to HiCache pools on this host. When set, - # it replaces cgroup and psutil discovery; the 10 GiB reserve and the - # per-local-rank split still apply. - SGLANG_HICACHE_HOST_MEMORY_BYTES = EnvInt(None) # Base token count for each MLA/DSA dedup broadcast chunk. SGLANG_MLA_DEDUP_CHUNK_TOKENS = EnvInt(2048) SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None) diff --git a/python/sglang/srt/mem_cache/hicache_auto_size.py b/python/sglang/srt/mem_cache/hicache_auto_size.py index 094f8020d4..f97bc226fc 100644 --- a/python/sglang/srt/mem_cache/hicache_auto_size.py +++ b/python/sglang/srt/mem_cache/hicache_auto_size.py @@ -93,7 +93,7 @@ def auto_size_hicache( return requested = get_memory().hicache_ratio device_bytes = _estimate_hicache_bytes(params, draft_plan) - budget = int(host_memory_budget_bytes() * fraction) + budget = int(host_memory_budget_bytes(auto_size=True) * fraction) ratio = min(requested, budget * (1 - _ALLOCATION_SLACK_FRACTION) / device_bytes) # One collective before any pool is built: PP stages own different pool # counts, so a per-pool collective could deadlock. diff --git a/python/sglang/srt/mem_cache/host_memory.py b/python/sglang/srt/mem_cache/host_memory.py index 5032c3e9c1..3ff7eb5acb 100644 --- a/python/sglang/srt/mem_cache/host_memory.py +++ b/python/sglang/srt/mem_cache/host_memory.py @@ -2,34 +2,24 @@ from __future__ import annotations -import functools import logging import re from pathlib import Path, PurePosixPath import psutil -from sglang.srt.environ import envs - logger = logging.getLogger(__name__) -_BUDGET_ENV = "SGLANG_HICACHE_HOST_MEMORY_BYTES" - - -@functools.cache -def _warn_no_cgroup_mount() -> None: - logger.warning( - "The process belongs to a cgroup but no cgroup filesystem is mounted; " - "sizing HiCache against host available memory. If the process runs " - "under a memory limit, set %s to its host memory budget.", - _BUDGET_ENV, - ) - def _unescape_mount_path(value: str) -> str: return re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value) +def _memory_stat(directory: Path) -> dict[str, int]: + lines = (directory / "memory.stat").read_text().splitlines() + return {key: int(value) for key, value in map(str.split, lines)} + + def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None: memberships = {} try: @@ -47,11 +37,9 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None: headroom = None resolved = False - cgroup_mounted = False for line in mounts.splitlines(): before, after = line.split(" - ", 1) filesystem, _, options = after.split()[:3] - cgroup_mounted |= filesystem in ("cgroup", "cgroup2") if filesystem not in memberships: continue if filesystem == "cgroup" and "memory" not in options.split(","): @@ -82,6 +70,13 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None: usage_name = ( "memory.current" if filesystem == "cgroup2" else "memory.usage_in_bytes" ) + # File LRU pages, as MemAvailable counts; shmem sits on the anon LRU. + # v1's unprefixed counters omit descendants, unlike its usage. + page_cache_keys = ( + ("active_file", "inactive_file") + if filesystem == "cgroup2" + else ("total_active_file", "total_inactive_file") + ) while True: for name in limits: try: @@ -95,38 +90,40 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None: # Do not silently ignore an unreadable usage file for a known # limit: falling back to host RAM could overrun the container. usage = int((directory / usage_name).read_text()) - remaining = max(0, limit - usage) + stat = _memory_stat(directory) + page_cache = sum(stat[key] for key in page_cache_keys) + # Charged page cache is reclaimed under the limit, not used. + remaining = max(0, limit - max(0, usage - page_cache)) headroom = remaining if headroom is None else min(headroom, remaining) if directory == mount: break directory = directory.parent if memberships and not resolved: - if not cgroup_mounted: - # E.g. a container sharing the host cgroup namespace without - # cgroupfs: no limit is visible, so size against host memory. - _warn_no_cgroup_mount() - return None - # A mounted but unmatched hierarchy may hide a real limit. raise RuntimeError( - "Cannot locate the process memory cgroup in mounted cgroup " - f"filesystems; set {_BUDGET_ENV} to the host memory budget to skip " - "cgroup discovery" + "Cannot locate the process memory cgroup in mounted cgroup filesystems" ) return headroom -def available_host_memory_bytes() -> int: - """Conservative allocatable RAM; charged file cache is not assumed reclaimable. - - An explicit SGLANG_HICACHE_HOST_MEMORY_BYTES replaces both probes. - """ - budget = envs.SGLANG_HICACHE_HOST_MEMORY_BYTES.get() - if budget is not None: - if budget <= 0: - raise ValueError(f"{_BUDGET_ENV} must be positive, got {budget}") - return budget +def available_host_memory_bytes(*, allow_cgroup_fallback: bool = False) -> int: + """Allocatable RAM, counting reclaimable page cache as free.""" available = psutil.virtual_memory().available - cgroup_headroom = _cgroup_memory_headroom() + try: + cgroup_headroom = _cgroup_memory_headroom() + except (OSError, RuntimeError) as exc: + if not allow_cgroup_fallback: + raise RuntimeError( + "Cannot automatically size HiCache host memory; " + "set --hicache-size explicitly (GB per rank), leaving room " + "for other memory use within the container limit." + ) from exc + logger.warning( + "Cannot determine cgroup memory headroom: %s. Checking explicitly " + "sized HiCache pools against host available memory; ensure all " + "local ranks' pools fit within the container memory limit.", + exc, + ) + return available if cgroup_headroom is not None: logger.info( "HiCache memory headroom: host %.1f GiB, cgroup %.1f GiB", diff --git a/python/sglang/srt/mem_cache/pool_host/base.py b/python/sglang/srt/mem_cache/pool_host/base.py index bb8b906659..8469bd28d6 100644 --- a/python/sglang/srt/mem_cache/pool_host/base.py +++ b/python/sglang/srt/mem_cache/pool_host/base.py @@ -82,12 +82,16 @@ def ranks_per_host() -> int: return max(launch_world_size // get_parallel().nnodes, 1) -def host_memory_budget_bytes(requested_bytes: int = 0) -> int: +def host_memory_budget_bytes( + requested_bytes: int = 0, *, auto_size: bool = False +) -> int: """Host RAM this rank may claim for a HiCache pool. Bound machine availability by the visible cgroup limits before splitting among local ranks. Independent engines with separate container budgets therefore size against their own remaining allowance. + Auto-sizing requires successful cgroup discovery; explicit pool sizing + can fall back to checking host availability when discovery fails. Inside host_memory_budget_scope, requested_bytes is booked against the snapshot when it fits; the allowance before booking is returned. @@ -98,7 +102,10 @@ def host_memory_budget_bytes(requested_bytes: int = 0) -> int: _host_memory_budget.set(available - requested_bytes) return available - free = available_host_memory_bytes() - HICACHE_HOST_MEMORY_RESERVE_BYTES + free = ( + available_host_memory_bytes(allow_cgroup_fallback=not auto_size) + - HICACHE_HOST_MEMORY_RESERVE_BYTES + ) return free // ranks_per_host() diff --git a/test/registered/unit/mem_cache/test_host_memory.py b/test/registered/unit/mem_cache/test_host_memory.py index e3e4b022f0..35f0683920 100644 --- a/test/registered/unit/mem_cache/test_host_memory.py +++ b/test/registered/unit/mem_cache/test_host_memory.py @@ -6,7 +6,6 @@ import unittest from pathlib import Path from unittest.mock import Mock, patch -from sglang.srt.environ import envs from sglang.srt.mem_cache import host_memory from sglang.test.ci.ci_register import register_cpu_ci @@ -35,7 +34,7 @@ class TestHostMemory(unittest.TestCase): f"1 0 0:1 {mount_root} {escaped} rw - {filesystem} cgroup {options}\n" ) - def memory(self, path, usage, maximum="max", high="max", v1=False): + def memory(self, path, usage, maximum="max", high="max", v1=False, stat=None): directory = self.mount / path directory.mkdir(parents=True, exist_ok=True) files = ( @@ -43,6 +42,11 @@ class TestHostMemory(unittest.TestCase): if v1 else {"memory.max": maximum, "memory.high": high, "memory.current": usage} ) + prefix = "total_" if v1 else "" + stat = {f"{prefix}active_file": 0, f"{prefix}inactive_file": 0} | (stat or {}) + files["memory.stat"] = "".join( + f"{key} {value}\n" for key, value in stat.items() + ) for name, value in files.items(): (directory / name).write_text(str(value)) @@ -78,6 +82,32 @@ class TestHostMemory(unittest.TestCase): self.memory("task", 400, 1000, v1=True) self.assertEqual(host_memory._cgroup_memory_headroom(self.proc), 600) + def test_reclaimable_page_cache_is_not_used(self): + """Page cache charged to the cgroup, such as a checkpoint just read, is + reclaimed under the limit and must not shrink the headroom. Shared + memory is counted in file/cache but cannot be reclaimed.""" + v2 = {"file": 700, "shmem": 200, "active_file": 200, "inactive_file": 300} + v1 = { + "cache": 700, + "shmem": 200, + "active_file": 0, + "inactive_file": 0, + "total_active_file": 200, + "total_inactive_file": 300, + } + for is_v1, stat, usage, expected in [ + (False, v2, 900, 600), + (True, v1, 900, 600), + # v1 usage is approximate and may trail the cache counters. + (True, v1, 400, 1000), + ]: + with self.subTest(v1=is_v1, usage=usage): + self.configure(v1=is_v1) + self.memory("task/engine", usage, 1000, v1=is_v1, stat=stat) + self.assertEqual( + host_memory._cgroup_memory_headroom(self.proc), expected + ) + def test_independent_engines_have_separate_allowances(self): # Both engines see the same host RAM but have different charged usage. for task, usage, expected in [("a", 300, 700), ("b", 600, 400)]: @@ -102,9 +132,14 @@ class TestHostMemory(unittest.TestCase): ) def test_host_availability_is_also_a_bound(self): - for cgroup, expected in [(None, 100), (200, 100), (50, 50)]: + for cgroup, allow_fallback, expected in [ + (None, False, 100), + (200, False, 100), + (50, False, 50), + (50, True, 50), + ]: with ( - self.subTest(cgroup=cgroup), + self.subTest(cgroup=cgroup, allow_fallback=allow_fallback), patch.object( host_memory.psutil, "virtual_memory", @@ -114,9 +149,14 @@ class TestHostMemory(unittest.TestCase): host_memory, "_cgroup_memory_headroom", return_value=cgroup ), ): - self.assertEqual(host_memory.available_host_memory_bytes(), expected) + self.assertEqual( + host_memory.available_host_memory_bytes( + allow_cgroup_fallback=allow_fallback + ), + expected, + ) - def available(self, host_available=5000): + def available(self, host_available=5000, *, allow_cgroup_fallback=False): """Public sizing entry point over the synthetic procfs.""" with ( patch.object( @@ -130,41 +170,36 @@ class TestHostMemory(unittest.TestCase): functools.partial(_cgroup_memory_headroom, self.proc), ), ): - return host_memory.available_host_memory_bytes() + return host_memory.available_host_memory_bytes( + allow_cgroup_fallback=allow_cgroup_fallback + ) def without_cgroupfs(self, v1=False): # A container sharing the host cgroup namespace, no cgroupfs mounted. self.configure("/system.slice/engine.scope", v1=v1) (self.proc / "self/mountinfo").write_text("1 0 0:1 / /proc rw - proc proc rw\n") - def test_no_cgroup_filesystem_sizes_against_host(self): + def test_unavailable_cgroup_requires_explicit_sizing(self): for v1 in [False, True]: - with self.subTest(v1=v1): - self.without_cgroupfs(v1=v1) - self.assertEqual(self.available(host_available=5000), 5000) + for mounted in [False, True]: + with self.subTest(v1=v1, mounted=mounted): + if mounted: + self.configure("/system.slice/engine.scope", v1=v1) + else: + self.without_cgroupfs(v1=v1) + with self.assertRaisesRegex(RuntimeError, "set --hicache-size"): + self.available() + self.assertEqual(self.available(allow_cgroup_fallback=True), 5000) - def test_explicit_budget_replaces_discovery(self): - # Mounted but unresolvable: discovery alone would raise. - self.configure("/system.slice/engine.scope") - with envs.SGLANG_HICACHE_HOST_MEMORY_BYTES.override(1234): - self.assertEqual(self.available(host_available=5000), 1234) - with envs.SGLANG_HICACHE_HOST_MEMORY_BYTES.override(0): - with self.assertRaisesRegex(ValueError, "must be positive"): - self.available() - - def test_unresolved_mounted_cgroup_fails(self): - self.configure("/system.slice/engine.scope") - with self.assertRaisesRegex( - RuntimeError, "Cannot locate.*SGLANG_HICACHE_HOST_MEMORY_BYTES" - ): - self.available() - - def test_missing_usage_for_known_limit_fails(self): - self.configure() - self.memory("task/engine", 100, 1000) - (self.mount / "task/engine/memory.current").unlink() - with self.assertRaises(FileNotFoundError): - host_memory._cgroup_memory_headroom(self.proc) + def test_missing_counters_for_known_limit_requires_explicit_sizing(self): + for name in ("memory.current", "memory.stat"): + with self.subTest(name=name): + self.configure() + self.memory("task/engine", 100, 1000) + (self.mount / "task/engine" / name).unlink() + with self.assertRaisesRegex(RuntimeError, "set --hicache-size"): + self.available() + self.assertEqual(self.available(allow_cgroup_fallback=True), 5000) if __name__ == "__main__":