[HiCache] Keep protected cgroup page cache charged in host budgets (#42788)
Co-authored-by: Ke Bao <ispobaoke@gmail.com>
This commit is contained in:
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path, PurePosixPath
|
||||
from stat import S_ISDIR
|
||||
|
||||
import psutil
|
||||
|
||||
@@ -20,6 +21,23 @@ def _memory_stat(directory: Path) -> dict[str, int]:
|
||||
return {key: int(value) for key, value in map(str.split, lines)}
|
||||
|
||||
|
||||
def _has_protected_descendants(directory: Path) -> bool:
|
||||
"""Treat configured or unreadable descendant memory.min as protected."""
|
||||
pending = [directory]
|
||||
try:
|
||||
while pending:
|
||||
for child in pending.pop().iterdir():
|
||||
if not S_ISDIR(child.stat().st_mode):
|
||||
continue
|
||||
if int((child / "memory.min").read_text()) != 0:
|
||||
return True
|
||||
pending.append(child)
|
||||
except (OSError, ValueError):
|
||||
# Unknown protection only disables cache credit, not the cgroup bound.
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
memberships = {}
|
||||
try:
|
||||
@@ -78,6 +96,7 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
else ("total_active_file", "total_inactive_file")
|
||||
)
|
||||
while True:
|
||||
usage = None
|
||||
for name in limits:
|
||||
try:
|
||||
value = (directory / name).read_text().strip()
|
||||
@@ -89,11 +108,20 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
limit = int(value)
|
||||
# Do not silently ignore an unreadable usage file for a known
|
||||
# limit: falling back to host RAM could overrun the container.
|
||||
usage = int((directory / usage_name).read_text())
|
||||
stat = _memory_stat(directory)
|
||||
page_cache = sum(stat[key] for key in page_cache_keys)
|
||||
# Charged page cache is reclaimed under the limit, not used.
|
||||
remaining = max(0, limit - max(0, usage - page_cache))
|
||||
if usage is None:
|
||||
usage = int((directory / usage_name).read_text())
|
||||
stat = _memory_stat(directory)
|
||||
page_cache = sum(stat[key] for key in page_cache_keys)
|
||||
# Protection is relative to the reclaim target: its own min
|
||||
# does not prevent reclaim, but descendants' min can.
|
||||
if (
|
||||
filesystem == "cgroup2"
|
||||
and page_cache > 0
|
||||
and _has_protected_descendants(directory)
|
||||
):
|
||||
page_cache = 0
|
||||
usage = max(0, usage - page_cache)
|
||||
remaining = max(0, limit - usage)
|
||||
headroom = remaining if headroom is None else min(headroom, remaining)
|
||||
if directory == mount:
|
||||
break
|
||||
|
||||
@@ -34,13 +34,20 @@ class TestHostMemory(unittest.TestCase):
|
||||
f"1 0 0:1 {mount_root} {escaped} rw - {filesystem} cgroup {options}\n"
|
||||
)
|
||||
|
||||
def memory(self, path, usage, maximum="max", high="max", v1=False, stat=None):
|
||||
def memory(
|
||||
self, path, usage, maximum="max", high="max", v1=False, stat=None, minimum=0
|
||||
):
|
||||
directory = self.mount / path
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
files = (
|
||||
{"memory.limit_in_bytes": maximum, "memory.usage_in_bytes": usage}
|
||||
if v1
|
||||
else {"memory.max": maximum, "memory.high": high, "memory.current": usage}
|
||||
else {
|
||||
"memory.max": maximum,
|
||||
"memory.high": high,
|
||||
"memory.current": usage,
|
||||
"memory.min": minimum,
|
||||
}
|
||||
)
|
||||
prefix = "total_" if v1 else ""
|
||||
stat = {f"{prefix}active_file": 0, f"{prefix}inactive_file": 0} | (stat or {})
|
||||
@@ -108,6 +115,17 @@ class TestHostMemory(unittest.TestCase):
|
||||
host_memory._cgroup_memory_headroom(self.proc), expected
|
||||
)
|
||||
|
||||
def test_protected_sibling_cache_stays_charged(self):
|
||||
self.configure()
|
||||
self.memory("task", 850, 900, stat={"inactive_file": 600})
|
||||
self.memory("task/engine", 250, 1000)
|
||||
self.memory("task/sibling", 600, stat={"inactive_file": 600})
|
||||
self.assertEqual(self.available(), 650)
|
||||
|
||||
# Parent reclaim must preserve the sibling's protected cache.
|
||||
(self.mount / "task/sibling/memory.min").write_text("600")
|
||||
self.assertEqual(self.available(), 50)
|
||||
|
||||
def test_independent_engines_have_separate_allowances(self):
|
||||
# Both engines see the same host RAM but have different charged usage.
|
||||
for task, usage, expected in [("a", 300, 700), ("b", 600, 400)]:
|
||||
@@ -189,7 +207,10 @@ class TestHostMemory(unittest.TestCase):
|
||||
self.without_cgroupfs(v1=v1)
|
||||
with self.assertRaisesRegex(RuntimeError, "set --hicache-size"):
|
||||
self.available()
|
||||
self.assertEqual(self.available(allow_cgroup_fallback=True), 5000)
|
||||
with self.assertLogs(host_memory.logger, level="WARNING"):
|
||||
self.assertEqual(
|
||||
self.available(allow_cgroup_fallback=True), 5000
|
||||
)
|
||||
|
||||
def test_missing_counters_for_known_limit_requires_explicit_sizing(self):
|
||||
for name in ("memory.current", "memory.stat"):
|
||||
@@ -199,7 +220,8 @@ class TestHostMemory(unittest.TestCase):
|
||||
(self.mount / "task/engine" / name).unlink()
|
||||
with self.assertRaisesRegex(RuntimeError, "set --hicache-size"):
|
||||
self.available()
|
||||
self.assertEqual(self.available(allow_cgroup_fallback=True), 5000)
|
||||
with self.assertLogs(host_memory.logger, level="WARNING"):
|
||||
self.assertEqual(self.available(allow_cgroup_fallback=True), 5000)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user