fix(nccl): disable graph buffer registration for pausable graph pools (#40648)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
a71c7d19e7
commit
dcf4e18283
@@ -336,12 +336,29 @@ def handle_data_parallelism(server_args: Any):
|
||||
run_post_process_pass(server_args, _tp_lm_head_all_to_all_default)
|
||||
run_post_process_pass(server_args, _dp_lm_head_validation)
|
||||
if resolving_view(server_args).enable_tp_lm_head_all_to_all:
|
||||
_disable_nccl_graph_buffer_registration()
|
||||
_disable_nccl_graph_buffer_registration(
|
||||
"the graph-captured TP LM-head all-to-all can deadlock with "
|
||||
"registered buffers"
|
||||
)
|
||||
if _graph_pool_is_pausable(server_args):
|
||||
_disable_nccl_graph_buffer_registration(
|
||||
"graph replay can hang once torch_memory_saver resume remaps the "
|
||||
"graph pool, leaving the capture-time registrations on released pages"
|
||||
)
|
||||
|
||||
|
||||
def _disable_nccl_graph_buffer_registration() -> None:
|
||||
"""Keep NCCL from registering the buffers of the graph-captured PyNccl
|
||||
all-to-all.
|
||||
def _graph_pool_is_pausable(server_args: Any) -> bool:
|
||||
"""Whether CUDA graphs are captured into torch_memory_saver memory, which
|
||||
release/resume of the `cuda_graph` tag remaps to new physical pages at the
|
||||
same virtual addresses."""
|
||||
return bool(
|
||||
resolving_view(server_args).enable_memory_saver
|
||||
and envs.SGLANG_MEMORY_SAVER_CUDA_GRAPH.get()
|
||||
)
|
||||
|
||||
|
||||
def _disable_nccl_graph_buffer_registration(reason: str) -> None:
|
||||
"""Keep NCCL from registering the buffers of graph-captured collectives.
|
||||
|
||||
NCCL_GRAPH_REGISTER (default on) registers the send/recv buffers of every
|
||||
collective captured in a CUDA graph for the lifetime of the graph, and
|
||||
@@ -353,15 +370,21 @@ def _disable_nccl_graph_buffer_registration() -> None:
|
||||
spin in ncclDevKernel_SendRecv forever, and every DP rank hangs.
|
||||
Reproduced on tp4/dp4/ep4 and on a multi-node tp16/dp16/ep16 PD decode
|
||||
deployment; disabling the registration removes the hang while dedicated
|
||||
all-to-all buffers alone do not. Must run before the schedulers create
|
||||
their NCCL communicators, which inherit this environment. An explicit
|
||||
setting wins.
|
||||
all-to-all buffers alone do not.
|
||||
|
||||
A pausable graph pool breaks the registrations too: torch_memory_saver
|
||||
resume keeps the pool's virtual addresses but maps new physical pages,
|
||||
while NCCL's capture-time registrations still point at the released pages,
|
||||
so graph replay can hang (fzyzcjy/torch_memory_saver#88).
|
||||
|
||||
Must run before the schedulers create their NCCL communicators, which
|
||||
inherit this environment. An explicit setting wins.
|
||||
"""
|
||||
if os.environ.setdefault("NCCL_GRAPH_REGISTER", "0") != "0":
|
||||
logger.warning(
|
||||
"NCCL_GRAPH_REGISTER=%s was set explicitly; the graph-captured TP "
|
||||
"LM-head all-to-all can deadlock with registered buffers.",
|
||||
"NCCL_GRAPH_REGISTER=%s was set explicitly; %s.",
|
||||
os.environ["NCCL_GRAPH_REGISTER"],
|
||||
reason,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user