[misc] Remove break-graph debug log; reclaim pid-less /dev/shm leaks in CI (#33929)

python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/breakable_cuda_graph.py

@@ -22,7 +22,6 @@ tensors remain valid across replays — we don't need Python-managed bridge
 buffers to keep break-point tensors at stable addresses.
 """
 
-import logging
 import threading
 from contextvars import ContextVar
 from typing import Any, Callable, Optional
@@ -39,8 +38,6 @@ from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.cuda_ut
 )
 from sglang.srt.utils import get_device_module, is_hip, is_xpu
 
-logger = logging.getLogger(__name__)
-
 _is_xpu = is_xpu()
 
 __all__ = [
@@ -226,8 +223,6 @@ def eager_on_graph(enable: bool, capture_stub: Optional[Callable] = None):
             if capture is None:
                 return inner(*args, **kwargs)
 
-            logger.debug("Break graph due to function: %s", inner.__name__)
-
             # End the segment that captured up to this break point.
             capture._end_current_segment()
 

python/sglang/srt/utils/stale_shm_cleanup.py

@@ -4,10 +4,10 @@ SGLang processes are torn down with SIGKILL (kill_process_tree, PDEATHSIG),
 which skips every Python-level unlink path, so /dev/shm segments accumulate
 until the tmpfs is full and the next scheduler init dies with SIGBUS.
 
-Segments created through make_shm_name() embed the creator pid, which lets a
-later server startup safely unlink segments whose creator is gone. The sweep
-only runs in CI (single-tenant runner containers); on shared dev machines a
-pid check against another user's process is not authoritative, so we skip.
+Pid-stamped names (see _creator_pid) are unlinked once their creator is dead;
+pid-less families (_ORPHAN_PREFIXES) are unlinked unconditionally, safe only
+because the sweep runs at CI job start right after killall.py. CI-only
+(SGLANG_IS_IN_CI): both rules assume a single-tenant runner container.
 """
 
 import logging
@@ -20,10 +20,16 @@ logger = logging.getLogger(__name__)
 _SHM_DIR = Path("/dev/shm")
 _SGL_SHM_PREFIX = "sgl_shm"
 
+_ORPHAN_PREFIXES = (
+    "sglang_loads_",  # managers/load_snapshot.py slot files
+    "cuda.shm.",  # CUDA IPC segments
+    "nccl-",  # NCCL communicator segments
+    "sem.loky-",  # loky/joblib semaphores
+)
+
 
 def make_shm_name(kind: str) -> str:
-    """Name a shared-memory segment so cleanup_stale_shm can identify and
-    reclaim it after its creator process dies: sgl_shm_<kind>_<pid>_<rand>."""
+    """Pid-stamped name (sgl_shm_<kind>_<pid>_<rand>) the sweep can reclaim."""
     return f"{_SGL_SHM_PREFIX}_{kind}_{os.getpid()}_{uuid.uuid4().hex[:8]}"
 
 
@@ -60,11 +66,10 @@ def _pid_alive(pid: int) -> bool:
 
 
 def cleanup_stale_shm() -> None:
-    """Unlink shared-memory segments whose creator process is dead.
+    """Unlink leaked shared-memory segments (rules in module docstring).
 
-    CI-only: gated on SGLANG_IS_IN_CI because the pid-liveness check is only
-    trustworthy when the container runs one job at a time. Best-effort: never
-    raises, since a failed sweep must not block server startup.
+    Best-effort: never raises, since a failed sweep must not block server
+    startup.
     """
     try:
         _cleanup_stale_shm_impl()
@@ -75,9 +80,8 @@ def cleanup_stale_shm() -> None:
 
 
 def _is_in_ci() -> bool:
-    # Read the env var directly (same semantics as sglang.utils.is_in_ci) so
-    # this module stays import-free and runnable by path from CI scripts
-    # before sglang is installed.
+    # Same semantics as sglang.utils.is_in_ci, read directly so the module
+    # stays import-free (CI runs it by path before sglang is installed).
     return os.environ.get("SGLANG_IS_IN_CI", "false").lower() in ("true", "1")
 
 
@@ -96,10 +100,13 @@ def _cleanup_stale_shm_impl() -> None:
         return
     for entry in entries:
         pid = _creator_pid(entry.name)
-        if pid is None or pid == os.getpid() or _pid_alive(pid):
+        if pid is not None:
             # A recycled pid reads as alive, so pid-reuse degrades to
             # under-collection (segment leaks), never to deleting a live
             # segment. Keep that bias when changing this check.
+            if pid == os.getpid() or _pid_alive(pid):
+                continue
+        elif not entry.name.startswith(_ORPHAN_PREFIXES):
             continue
         try:
             size = entry.stat().st_size

test/registered/utils/test_stale_shm_cleanup.py

@@ -44,13 +44,13 @@ class TestCleanupStaleShm(unittest.TestCase):
     def _make_segment(self, name: str) -> str:
         shm = shared_memory.SharedMemory(create=True, size=4096, name=name)
         shm.close()
-        self.addCleanup(self._unlink_quiet, name)
+        self.addCleanup(self._unlink_quiet, f"/dev/shm/{name}")
         return name
 
     @staticmethod
-    def _unlink_quiet(name: str):
+    def _unlink_quiet(path: str):
         try:
-            shared_memory.SharedMemory(name=name).unlink()
+            os.unlink(path)
         except FileNotFoundError:
             pass
 
@@ -121,6 +121,33 @@ class TestCleanupStaleShm(unittest.TestCase):
 
         self.assertFalse(os.path.exists(f"/dev/shm/{stale}"))
 
+    def _make_raw_file(self, name: str) -> str:
+        """Orphan families are plain files, not shared_memory segments."""
+        path = f"/dev/shm/{name}"
+        with open(path, "wb") as f:
+            f.write(b"\0" * 4096)
+        self.addCleanup(self._unlink_quiet, path)
+        return path
+
+    @unittest.skipUnless(
+        os.environ.get("SGLANG_IS_IN_CI", "").lower() in ("true", "1"),
+        "sweeps orphan families unconditionally; only safe on a CI runner",
+    )
+    def test_orphan_family_sweep(self):
+        stale = [
+            self._make_raw_file("sglang_loads_test_deadbeef.shm"),
+            self._make_raw_file("cuda.shm.0.deadbeef.1"),
+            self._make_raw_file("nccl-testonly"),
+            self._make_raw_file("sem.loky-0-testonly"),
+        ]
+        unknown = self._make_raw_file("unknown_family_file")
+
+        cleanup_stale_shm()
+
+        for path in stale:
+            self.assertFalse(os.path.exists(path), path)
+        self.assertTrue(os.path.exists(unknown))
+
 
 if __name__ == "__main__":
     unittest.main()