diff --git a/python/sglang/srt/arg_groups/parallel_hook.py b/python/sglang/srt/arg_groups/parallel_hook.py index 06101adfd9..50ee8da513 100644 --- a/python/sglang/srt/arg_groups/parallel_hook.py +++ b/python/sglang/srt/arg_groups/parallel_hook.py @@ -336,12 +336,29 @@ def handle_data_parallelism(server_args: Any): run_post_process_pass(server_args, _tp_lm_head_all_to_all_default) run_post_process_pass(server_args, _dp_lm_head_validation) if resolving_view(server_args).enable_tp_lm_head_all_to_all: - _disable_nccl_graph_buffer_registration() + _disable_nccl_graph_buffer_registration( + "the graph-captured TP LM-head all-to-all can deadlock with " + "registered buffers" + ) + if _graph_pool_is_pausable(server_args): + _disable_nccl_graph_buffer_registration( + "graph replay can hang once torch_memory_saver resume remaps the " + "graph pool, leaving the capture-time registrations on released pages" + ) -def _disable_nccl_graph_buffer_registration() -> None: - """Keep NCCL from registering the buffers of the graph-captured PyNccl - all-to-all. +def _graph_pool_is_pausable(server_args: Any) -> bool: + """Whether CUDA graphs are captured into torch_memory_saver memory, which + release/resume of the `cuda_graph` tag remaps to new physical pages at the + same virtual addresses.""" + return bool( + resolving_view(server_args).enable_memory_saver + and envs.SGLANG_MEMORY_SAVER_CUDA_GRAPH.get() + ) + + +def _disable_nccl_graph_buffer_registration(reason: str) -> None: + """Keep NCCL from registering the buffers of graph-captured collectives. NCCL_GRAPH_REGISTER (default on) registers the send/recv buffers of every collective captured in a CUDA graph for the lifetime of the graph, and @@ -353,15 +370,21 @@ def _disable_nccl_graph_buffer_registration() -> None: spin in ncclDevKernel_SendRecv forever, and every DP rank hangs. Reproduced on tp4/dp4/ep4 and on a multi-node tp16/dp16/ep16 PD decode deployment; disabling the registration removes the hang while dedicated - all-to-all buffers alone do not. Must run before the schedulers create - their NCCL communicators, which inherit this environment. An explicit - setting wins. + all-to-all buffers alone do not. + + A pausable graph pool breaks the registrations too: torch_memory_saver + resume keeps the pool's virtual addresses but maps new physical pages, + while NCCL's capture-time registrations still point at the released pages, + so graph replay can hang (fzyzcjy/torch_memory_saver#88). + + Must run before the schedulers create their NCCL communicators, which + inherit this environment. An explicit setting wins. """ if os.environ.setdefault("NCCL_GRAPH_REGISTER", "0") != "0": logger.warning( - "NCCL_GRAPH_REGISTER=%s was set explicitly; the graph-captured TP " - "LM-head all-to-all can deadlock with registered buffers.", + "NCCL_GRAPH_REGISTER=%s was set explicitly; %s.", os.environ["NCCL_GRAPH_REGISTER"], + reason, )