[HiCache] Fix cgroup page-cache accounting and the sizing fallback when cgroup discovery fails (#42420)
Co-authored-by: merrymercy <merrymercy@users.noreply.github.com>
Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
Co-authored-by: nvjullin <nvjullin@users.noreply.github.com>
(cherry picked from commit 4ab720e655)
This commit is contained in:
@@ -91,7 +91,7 @@ def auto_size_hicache(
|
||||
return
|
||||
requested = get_memory().hicache_ratio
|
||||
device_bytes = _estimate_hicache_bytes(params, draft_plan)
|
||||
budget = int(host_memory_budget_bytes() * fraction)
|
||||
budget = int(host_memory_budget_bytes(auto_size=True) * fraction)
|
||||
ratio = min(requested, budget * (1 - _ALLOCATION_SLACK_FRACTION) / device_bytes)
|
||||
# One collective before any pool is built: PP stages own different pool
|
||||
# counts, so a per-pool collective could deadlock.
|
||||
|
||||
@@ -15,6 +15,11 @@ def _unescape_mount_path(value: str) -> str:
|
||||
return re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)
|
||||
|
||||
|
||||
def _memory_stat(directory: Path) -> dict[str, int]:
|
||||
lines = (directory / "memory.stat").read_text().splitlines()
|
||||
return {key: int(value) for key, value in map(str.split, lines)}
|
||||
|
||||
|
||||
def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
memberships = {}
|
||||
try:
|
||||
@@ -65,6 +70,13 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
usage_name = (
|
||||
"memory.current" if filesystem == "cgroup2" else "memory.usage_in_bytes"
|
||||
)
|
||||
# File LRU pages, as MemAvailable counts; shmem sits on the anon LRU.
|
||||
# v1's unprefixed counters omit descendants, unlike its usage.
|
||||
page_cache_keys = (
|
||||
("active_file", "inactive_file")
|
||||
if filesystem == "cgroup2"
|
||||
else ("total_active_file", "total_inactive_file")
|
||||
)
|
||||
while True:
|
||||
for name in limits:
|
||||
try:
|
||||
@@ -78,7 +90,10 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
# Do not silently ignore an unreadable usage file for a known
|
||||
# limit: falling back to host RAM could overrun the container.
|
||||
usage = int((directory / usage_name).read_text())
|
||||
remaining = max(0, limit - usage)
|
||||
stat = _memory_stat(directory)
|
||||
page_cache = sum(stat[key] for key in page_cache_keys)
|
||||
# Charged page cache is reclaimed under the limit, not used.
|
||||
remaining = max(0, limit - max(0, usage - page_cache))
|
||||
headroom = remaining if headroom is None else min(headroom, remaining)
|
||||
if directory == mount:
|
||||
break
|
||||
@@ -90,10 +105,25 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
return headroom
|
||||
|
||||
|
||||
def available_host_memory_bytes() -> int:
|
||||
"""Conservative allocatable RAM; charged file cache is not assumed reclaimable."""
|
||||
def available_host_memory_bytes(*, allow_cgroup_fallback: bool = False) -> int:
|
||||
"""Allocatable RAM, counting reclaimable page cache as free."""
|
||||
available = psutil.virtual_memory().available
|
||||
cgroup_headroom = _cgroup_memory_headroom()
|
||||
try:
|
||||
cgroup_headroom = _cgroup_memory_headroom()
|
||||
except (OSError, RuntimeError) as exc:
|
||||
if not allow_cgroup_fallback:
|
||||
raise RuntimeError(
|
||||
"Cannot automatically size HiCache host memory; "
|
||||
"set --hicache-size explicitly (GB per rank), leaving room "
|
||||
"for other memory use within the container limit."
|
||||
) from exc
|
||||
logger.warning(
|
||||
"Cannot determine cgroup memory headroom: %s. Checking explicitly "
|
||||
"sized HiCache pools against host available memory; ensure all "
|
||||
"local ranks' pools fit within the container memory limit.",
|
||||
exc,
|
||||
)
|
||||
return available
|
||||
if cgroup_headroom is not None:
|
||||
logger.info(
|
||||
"HiCache memory headroom: host %.1f GiB, cgroup %.1f GiB",
|
||||
|
||||
@@ -61,12 +61,16 @@ def ranks_per_host() -> int:
|
||||
return max(launch_world_size // get_parallel().nnodes, 1)
|
||||
|
||||
|
||||
def host_memory_budget_bytes(requested_bytes: int = 0) -> int:
|
||||
def host_memory_budget_bytes(
|
||||
requested_bytes: int = 0, *, auto_size: bool = False
|
||||
) -> int:
|
||||
"""Host RAM this rank may claim for a HiCache pool.
|
||||
|
||||
Bound machine availability by the visible cgroup limits before splitting
|
||||
among local ranks. Independent engines with separate container budgets
|
||||
therefore size against their own remaining allowance.
|
||||
Auto-sizing requires successful cgroup discovery; explicit pool sizing
|
||||
can fall back to checking host availability when discovery fails.
|
||||
|
||||
Inside host_memory_budget_scope, requested_bytes is booked against the
|
||||
snapshot when it fits; the allowance before booking is returned.
|
||||
@@ -77,7 +81,10 @@ def host_memory_budget_bytes(requested_bytes: int = 0) -> int:
|
||||
_host_memory_budget.set(available - requested_bytes)
|
||||
return available
|
||||
|
||||
free = available_host_memory_bytes() - HICACHE_HOST_MEMORY_RESERVE_BYTES
|
||||
free = (
|
||||
available_host_memory_bytes(allow_cgroup_fallback=not auto_size)
|
||||
- HICACHE_HOST_MEMORY_RESERVE_BYTES
|
||||
)
|
||||
return free // ranks_per_host()
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Exercise container and ancestor budgets using synthetic procfs/cgroup files."""
|
||||
|
||||
import functools
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
@@ -8,6 +9,8 @@ from unittest.mock import Mock, patch
|
||||
from sglang.srt.mem_cache import host_memory
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
_cgroup_memory_headroom = host_memory._cgroup_memory_headroom
|
||||
|
||||
register_cpu_ci(est_time=1, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
@@ -31,7 +34,7 @@ class TestHostMemory(unittest.TestCase):
|
||||
f"1 0 0:1 {mount_root} {escaped} rw - {filesystem} cgroup {options}\n"
|
||||
)
|
||||
|
||||
def memory(self, path, usage, maximum="max", high="max", v1=False):
|
||||
def memory(self, path, usage, maximum="max", high="max", v1=False, stat=None):
|
||||
directory = self.mount / path
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
files = (
|
||||
@@ -39,6 +42,11 @@ class TestHostMemory(unittest.TestCase):
|
||||
if v1
|
||||
else {"memory.max": maximum, "memory.high": high, "memory.current": usage}
|
||||
)
|
||||
prefix = "total_" if v1 else ""
|
||||
stat = {f"{prefix}active_file": 0, f"{prefix}inactive_file": 0} | (stat or {})
|
||||
files["memory.stat"] = "".join(
|
||||
f"{key} {value}\n" for key, value in stat.items()
|
||||
)
|
||||
for name, value in files.items():
|
||||
(directory / name).write_text(str(value))
|
||||
|
||||
@@ -74,6 +82,32 @@ class TestHostMemory(unittest.TestCase):
|
||||
self.memory("task", 400, 1000, v1=True)
|
||||
self.assertEqual(host_memory._cgroup_memory_headroom(self.proc), 600)
|
||||
|
||||
def test_reclaimable_page_cache_is_not_used(self):
|
||||
"""Page cache charged to the cgroup, such as a checkpoint just read, is
|
||||
reclaimed under the limit and must not shrink the headroom. Shared
|
||||
memory is counted in file/cache but cannot be reclaimed."""
|
||||
v2 = {"file": 700, "shmem": 200, "active_file": 200, "inactive_file": 300}
|
||||
v1 = {
|
||||
"cache": 700,
|
||||
"shmem": 200,
|
||||
"active_file": 0,
|
||||
"inactive_file": 0,
|
||||
"total_active_file": 200,
|
||||
"total_inactive_file": 300,
|
||||
}
|
||||
for is_v1, stat, usage, expected in [
|
||||
(False, v2, 900, 600),
|
||||
(True, v1, 900, 600),
|
||||
# v1 usage is approximate and may trail the cache counters.
|
||||
(True, v1, 400, 1000),
|
||||
]:
|
||||
with self.subTest(v1=is_v1, usage=usage):
|
||||
self.configure(v1=is_v1)
|
||||
self.memory("task/engine", usage, 1000, v1=is_v1, stat=stat)
|
||||
self.assertEqual(
|
||||
host_memory._cgroup_memory_headroom(self.proc), expected
|
||||
)
|
||||
|
||||
def test_independent_engines_have_separate_allowances(self):
|
||||
# Both engines see the same host RAM but have different charged usage.
|
||||
for task, usage, expected in [("a", 300, 700), ("b", 600, 400)]:
|
||||
@@ -98,9 +132,14 @@ class TestHostMemory(unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_host_availability_is_also_a_bound(self):
|
||||
for cgroup, expected in [(None, 100), (200, 100), (50, 50)]:
|
||||
for cgroup, allow_fallback, expected in [
|
||||
(None, False, 100),
|
||||
(200, False, 100),
|
||||
(50, False, 50),
|
||||
(50, True, 50),
|
||||
]:
|
||||
with (
|
||||
self.subTest(cgroup=cgroup),
|
||||
self.subTest(cgroup=cgroup, allow_fallback=allow_fallback),
|
||||
patch.object(
|
||||
host_memory.psutil,
|
||||
"virtual_memory",
|
||||
@@ -110,20 +149,57 @@ class TestHostMemory(unittest.TestCase):
|
||||
host_memory, "_cgroup_memory_headroom", return_value=cgroup
|
||||
),
|
||||
):
|
||||
self.assertEqual(host_memory.available_host_memory_bytes(), expected)
|
||||
self.assertEqual(
|
||||
host_memory.available_host_memory_bytes(
|
||||
allow_cgroup_fallback=allow_fallback
|
||||
),
|
||||
expected,
|
||||
)
|
||||
|
||||
def test_unmounted_memory_cgroup_fails(self):
|
||||
self.configure()
|
||||
(self.proc / "self/mountinfo").write_text("")
|
||||
with self.assertRaisesRegex(RuntimeError, "Cannot locate"):
|
||||
host_memory._cgroup_memory_headroom(self.proc)
|
||||
def available(self, host_available=5000, *, allow_cgroup_fallback=False):
|
||||
"""Public sizing entry point over the synthetic procfs."""
|
||||
with (
|
||||
patch.object(
|
||||
host_memory.psutil,
|
||||
"virtual_memory",
|
||||
return_value=Mock(available=host_available),
|
||||
),
|
||||
patch.object(
|
||||
host_memory,
|
||||
"_cgroup_memory_headroom",
|
||||
functools.partial(_cgroup_memory_headroom, self.proc),
|
||||
),
|
||||
):
|
||||
return host_memory.available_host_memory_bytes(
|
||||
allow_cgroup_fallback=allow_cgroup_fallback
|
||||
)
|
||||
|
||||
def test_missing_usage_for_known_limit_fails(self):
|
||||
self.configure()
|
||||
self.memory("task/engine", 100, 1000)
|
||||
(self.mount / "task/engine/memory.current").unlink()
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
host_memory._cgroup_memory_headroom(self.proc)
|
||||
def without_cgroupfs(self, v1=False):
|
||||
# A container sharing the host cgroup namespace, no cgroupfs mounted.
|
||||
self.configure("/system.slice/engine.scope", v1=v1)
|
||||
(self.proc / "self/mountinfo").write_text("1 0 0:1 / /proc rw - proc proc rw\n")
|
||||
|
||||
def test_unavailable_cgroup_requires_explicit_sizing(self):
|
||||
for v1 in [False, True]:
|
||||
for mounted in [False, True]:
|
||||
with self.subTest(v1=v1, mounted=mounted):
|
||||
if mounted:
|
||||
self.configure("/system.slice/engine.scope", v1=v1)
|
||||
else:
|
||||
self.without_cgroupfs(v1=v1)
|
||||
with self.assertRaisesRegex(RuntimeError, "set --hicache-size"):
|
||||
self.available()
|
||||
self.assertEqual(self.available(allow_cgroup_fallback=True), 5000)
|
||||
|
||||
def test_missing_counters_for_known_limit_requires_explicit_sizing(self):
|
||||
for name in ("memory.current", "memory.stat"):
|
||||
with self.subTest(name=name):
|
||||
self.configure()
|
||||
self.memory("task/engine", 100, 1000)
|
||||
(self.mount / "task/engine" / name).unlink()
|
||||
with self.assertRaisesRegex(RuntimeError, "set --hicache-size"):
|
||||
self.available()
|
||||
self.assertEqual(self.available(allow_cgroup_fallback=True), 5000)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user