From afa79330b8d2f63934f721857f4462a1f4cc9a8a Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 6 Aug 2026 21:18:01 -0700 Subject: [PATCH] [misc] Remove break-graph debug log; reclaim pid-less /dev/shm leaks in CI (#33929) --- .../breakable_cuda_graph.py | 5 --- python/sglang/srt/utils/stale_shm_cleanup.py | 35 +++++++++++-------- .../utils/test_stale_shm_cleanup.py | 33 +++++++++++++++-- 3 files changed, 51 insertions(+), 22 deletions(-) diff --git a/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/breakable_cuda_graph.py b/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/breakable_cuda_graph.py index 9c7601515..59f4a2e6b 100644 --- a/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/breakable_cuda_graph.py +++ b/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/breakable_cuda_graph.py @@ -22,7 +22,6 @@ tensors remain valid across replays — we don't need Python-managed bridge buffers to keep break-point tensors at stable addresses. """ -import logging import threading from contextvars import ContextVar from typing import Any, Callable, Optional @@ -39,8 +38,6 @@ from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.cuda_ut ) from sglang.srt.utils import get_device_module, is_hip, is_xpu -logger = logging.getLogger(__name__) - _is_xpu = is_xpu() __all__ = [ @@ -226,8 +223,6 @@ def eager_on_graph(enable: bool, capture_stub: Optional[Callable] = None): if capture is None: return inner(*args, **kwargs) - logger.debug("Break graph due to function: %s", inner.__name__) - # End the segment that captured up to this break point. capture._end_current_segment() diff --git a/python/sglang/srt/utils/stale_shm_cleanup.py b/python/sglang/srt/utils/stale_shm_cleanup.py index 5a8fca0bf..885fd55ee 100644 --- a/python/sglang/srt/utils/stale_shm_cleanup.py +++ b/python/sglang/srt/utils/stale_shm_cleanup.py @@ -4,10 +4,10 @@ SGLang processes are torn down with SIGKILL (kill_process_tree, PDEATHSIG), which skips every Python-level unlink path, so /dev/shm segments accumulate until the tmpfs is full and the next scheduler init dies with SIGBUS. -Segments created through make_shm_name() embed the creator pid, which lets a -later server startup safely unlink segments whose creator is gone. The sweep -only runs in CI (single-tenant runner containers); on shared dev machines a -pid check against another user's process is not authoritative, so we skip. +Pid-stamped names (see _creator_pid) are unlinked once their creator is dead; +pid-less families (_ORPHAN_PREFIXES) are unlinked unconditionally, safe only +because the sweep runs at CI job start right after killall.py. CI-only +(SGLANG_IS_IN_CI): both rules assume a single-tenant runner container. """ import logging @@ -20,10 +20,16 @@ logger = logging.getLogger(__name__) _SHM_DIR = Path("/dev/shm") _SGL_SHM_PREFIX = "sgl_shm" +_ORPHAN_PREFIXES = ( + "sglang_loads_", # managers/load_snapshot.py slot files + "cuda.shm.", # CUDA IPC segments + "nccl-", # NCCL communicator segments + "sem.loky-", # loky/joblib semaphores +) + def make_shm_name(kind: str) -> str: - """Name a shared-memory segment so cleanup_stale_shm can identify and - reclaim it after its creator process dies: sgl_shm___.""" + """Pid-stamped name (sgl_shm___) the sweep can reclaim.""" return f"{_SGL_SHM_PREFIX}_{kind}_{os.getpid()}_{uuid.uuid4().hex[:8]}" @@ -60,11 +66,10 @@ def _pid_alive(pid: int) -> bool: def cleanup_stale_shm() -> None: - """Unlink shared-memory segments whose creator process is dead. + """Unlink leaked shared-memory segments (rules in module docstring). - CI-only: gated on SGLANG_IS_IN_CI because the pid-liveness check is only - trustworthy when the container runs one job at a time. Best-effort: never - raises, since a failed sweep must not block server startup. + Best-effort: never raises, since a failed sweep must not block server + startup. """ try: _cleanup_stale_shm_impl() @@ -75,9 +80,8 @@ def cleanup_stale_shm() -> None: def _is_in_ci() -> bool: - # Read the env var directly (same semantics as sglang.utils.is_in_ci) so - # this module stays import-free and runnable by path from CI scripts - # before sglang is installed. + # Same semantics as sglang.utils.is_in_ci, read directly so the module + # stays import-free (CI runs it by path before sglang is installed). return os.environ.get("SGLANG_IS_IN_CI", "false").lower() in ("true", "1") @@ -96,10 +100,13 @@ def _cleanup_stale_shm_impl() -> None: return for entry in entries: pid = _creator_pid(entry.name) - if pid is None or pid == os.getpid() or _pid_alive(pid): + if pid is not None: # A recycled pid reads as alive, so pid-reuse degrades to # under-collection (segment leaks), never to deleting a live # segment. Keep that bias when changing this check. + if pid == os.getpid() or _pid_alive(pid): + continue + elif not entry.name.startswith(_ORPHAN_PREFIXES): continue try: size = entry.stat().st_size diff --git a/test/registered/utils/test_stale_shm_cleanup.py b/test/registered/utils/test_stale_shm_cleanup.py index 810cc869d..ef8b60234 100644 --- a/test/registered/utils/test_stale_shm_cleanup.py +++ b/test/registered/utils/test_stale_shm_cleanup.py @@ -44,13 +44,13 @@ class TestCleanupStaleShm(unittest.TestCase): def _make_segment(self, name: str) -> str: shm = shared_memory.SharedMemory(create=True, size=4096, name=name) shm.close() - self.addCleanup(self._unlink_quiet, name) + self.addCleanup(self._unlink_quiet, f"/dev/shm/{name}") return name @staticmethod - def _unlink_quiet(name: str): + def _unlink_quiet(path: str): try: - shared_memory.SharedMemory(name=name).unlink() + os.unlink(path) except FileNotFoundError: pass @@ -121,6 +121,33 @@ class TestCleanupStaleShm(unittest.TestCase): self.assertFalse(os.path.exists(f"/dev/shm/{stale}")) + def _make_raw_file(self, name: str) -> str: + """Orphan families are plain files, not shared_memory segments.""" + path = f"/dev/shm/{name}" + with open(path, "wb") as f: + f.write(b"\0" * 4096) + self.addCleanup(self._unlink_quiet, path) + return path + + @unittest.skipUnless( + os.environ.get("SGLANG_IS_IN_CI", "").lower() in ("true", "1"), + "sweeps orphan families unconditionally; only safe on a CI runner", + ) + def test_orphan_family_sweep(self): + stale = [ + self._make_raw_file("sglang_loads_test_deadbeef.shm"), + self._make_raw_file("cuda.shm.0.deadbeef.1"), + self._make_raw_file("nccl-testonly"), + self._make_raw_file("sem.loky-0-testonly"), + ] + unknown = self._make_raw_file("unknown_family_file") + + cleanup_stale_shm() + + for path in stale: + self.assertFalse(os.path.exists(path), path) + self.assertTrue(os.path.exists(unknown)) + if __name__ == "__main__": unittest.main()