[Fix] nccl: Disable graph buffer registration whenever a captured graph can hold a collective (#43216)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
Jimmy Shong
parent
13999673ef
commit
786cdfc040
@@ -366,26 +366,37 @@ def handle_data_parallelism(server_args: Any):
|
||||
|
||||
run_post_process_pass(server_args, _tp_lm_head_all_to_all_default)
|
||||
run_post_process_pass(server_args, _dp_lm_head_validation)
|
||||
if resolving_view(server_args).enable_tp_lm_head_all_to_all:
|
||||
if _graph_may_capture_collectives(server_args):
|
||||
_disable_nccl_graph_buffer_registration(
|
||||
"the graph-captured TP LM-head all-to-all can deadlock with "
|
||||
"graph replay of captured cross-rank collectives can hang with "
|
||||
"registered buffers"
|
||||
)
|
||||
if _graph_pool_is_pausable(server_args):
|
||||
_disable_nccl_graph_buffer_registration(
|
||||
"graph replay can hang once torch_memory_saver resume remaps the "
|
||||
"graph pool, leaving the capture-time registrations on released pages"
|
||||
)
|
||||
|
||||
|
||||
def _graph_pool_is_pausable(server_args: Any) -> bool:
|
||||
"""Whether CUDA graphs are captured into torch_memory_saver memory, which
|
||||
release/resume of the `cuda_graph` tag remaps to new physical pages at the
|
||||
same virtual addresses."""
|
||||
return bool(
|
||||
resolving_view(server_args).enable_memory_saver
|
||||
and envs.SGLANG_MEMORY_SAVER_CUDA_GRAPH.get()
|
||||
)
|
||||
def _graph_may_capture_collectives(server_args: Any) -> bool:
|
||||
"""Whether any captured CUDA graph can hold a cross-rank collective.
|
||||
|
||||
Conservative workaround, root cause still open: multi-node TP16 decode
|
||||
replay hangs with NCCL_GRAPH_REGISTER on even with the TP LM-head
|
||||
all-to-all and the pausable graph pool both off, and stops hanging once
|
||||
the registration is off. Graph-pool address reuse is the suspected
|
||||
trigger, not a confirmed mechanism, so the guard keys on any multi-rank
|
||||
capture rather than on the feature flags that first exposed it. Checking
|
||||
every phase also keeps the prefill-into-pausable-pool case the old pool
|
||||
guard covered.
|
||||
|
||||
Errs toward True. Later passes can still narrow cuda_graph_config. On the
|
||||
reproducing deployment, a single 10-minute run showed no throughput
|
||||
difference with the registration off; other deployment shapes are
|
||||
unmeasured.
|
||||
"""
|
||||
view = resolving_view(server_args)
|
||||
if max(view.tp_size, view.ep_size, view.dp_size) <= 1:
|
||||
return False
|
||||
config = view.cuda_graph_config
|
||||
if config is None:
|
||||
return True
|
||||
return any(config[phase].backend != Backend.DISABLED for phase in Phase.ALL)
|
||||
|
||||
|
||||
def _disable_nccl_graph_buffer_registration(reason: str) -> None:
|
||||
|
||||
@@ -4543,14 +4543,14 @@ class TestRustServerDpAttentionControlBroadcast(unittest.TestCase):
|
||||
|
||||
|
||||
class TestTpLmHeadAllToAllNcclGraphRegister(unittest.TestCase):
|
||||
"""The graph-captured TP LM-head all-to-all must not run with NCCL's
|
||||
graph buffer registration: registered graph-pool temporaries deadlock the
|
||||
exchange under DP-rank ramps."""
|
||||
"""A decode graph that can capture a collective must not run with NCCL's
|
||||
graph buffer registration: the registered graph-pool temporaries alias
|
||||
tensors the pool hands out later, and replay then deadlocks."""
|
||||
|
||||
def _resolve(self, **kwargs):
|
||||
# The handler reads the DP-adjusted prefill knobs the pipeline would
|
||||
# have settled by then; the dummy-model pipeline itself returns early.
|
||||
server_args = ServerArgs(
|
||||
fields = dict(
|
||||
model_path="dummy",
|
||||
attn_dp_size=2,
|
||||
tp_size=2,
|
||||
@@ -4558,8 +4558,9 @@ class TestTpLmHeadAllToAllNcclGraphRegister(unittest.TestCase):
|
||||
cuda_graph_config=CudaGraphConfig(
|
||||
prefill=PhaseConfig(backend=Backend.DISABLED)
|
||||
),
|
||||
**kwargs,
|
||||
)
|
||||
fields.update(kwargs)
|
||||
server_args = ServerArgs(**fields)
|
||||
parallel_hook.handle_data_parallelism(server_args)
|
||||
return server_args
|
||||
|
||||
@@ -4579,9 +4580,9 @@ class TestTpLmHeadAllToAllNcclGraphRegister(unittest.TestCase):
|
||||
self.assertEqual(os.environ["NCCL_GRAPH_REGISTER"], "1")
|
||||
self.assertIn("NCCL_GRAPH_REGISTER=1", "\n".join(logs.output))
|
||||
|
||||
def test_without_all_to_all_env_is_untouched(self):
|
||||
# Unified serving keeps the all-to-all off by default, and a decode
|
||||
# node with the DP LM head never takes the all-to-all.
|
||||
def test_without_all_to_all_registration_still_defaults_off(self):
|
||||
# The hazard is the graph-captured collective, not the feature that
|
||||
# happens to issue it, so the default holds with the all-to-all off.
|
||||
for kwargs in (
|
||||
{},
|
||||
{"disaggregation_mode": "decode", "enable_dp_lm_head": True},
|
||||
@@ -4592,7 +4593,67 @@ class TestTpLmHeadAllToAllNcclGraphRegister(unittest.TestCase):
|
||||
self.assertFalse(
|
||||
resolution_result(server_args, "enable_tp_lm_head_all_to_all")
|
||||
)
|
||||
self.assertNotIn("NCCL_GRAPH_REGISTER", os.environ)
|
||||
self.assertEqual(os.environ.get("NCCL_GRAPH_REGISTER"), "0")
|
||||
|
||||
def test_plain_tp_defaults_off(self):
|
||||
# The motivating shape: plain TP with the all-to-all and memory saver off.
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("NCCL_GRAPH_REGISTER", None)
|
||||
server_args = self._resolve(
|
||||
attn_dp_size=1,
|
||||
tp_size=2,
|
||||
enable_tp_lm_head_all_to_all=False,
|
||||
enable_memory_saver=False,
|
||||
)
|
||||
self.assertFalse(
|
||||
resolution_result(server_args, "enable_tp_lm_head_all_to_all")
|
||||
)
|
||||
self.assertEqual(os.environ.get("NCCL_GRAPH_REGISTER"), "0")
|
||||
|
||||
def test_pausable_graph_pool_defaults_off(self):
|
||||
# Isolates the torch_memory_saver trigger: the all-to-all stays off, so
|
||||
# only the remapped graph pool can account for the default.
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("NCCL_GRAPH_REGISTER", None)
|
||||
server_args = self._resolve(enable_memory_saver=True)
|
||||
self.assertFalse(
|
||||
resolution_result(server_args, "enable_tp_lm_head_all_to_all")
|
||||
)
|
||||
self.assertEqual(os.environ.get("NCCL_GRAPH_REGISTER"), "0")
|
||||
|
||||
def test_prefill_only_capture_defaults_off(self):
|
||||
# Decode capture off, prefill still capturing into the same pausable
|
||||
# pool: the collective is in a graph either way, so the registration has
|
||||
# to go regardless of which phase holds it.
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("NCCL_GRAPH_REGISTER", None)
|
||||
self._resolve(
|
||||
enable_memory_saver=True,
|
||||
cuda_graph_config=CudaGraphConfig(
|
||||
decode=PhaseConfig(backend=Backend.DISABLED),
|
||||
prefill=PhaseConfig(backend=Backend.FULL),
|
||||
),
|
||||
)
|
||||
self.assertEqual(os.environ.get("NCCL_GRAPH_REGISTER"), "0")
|
||||
|
||||
def test_single_rank_env_is_untouched(self):
|
||||
# One rank issues no collective, so a capture registers nothing.
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("NCCL_GRAPH_REGISTER", None)
|
||||
self._resolve(attn_dp_size=1, tp_size=1)
|
||||
self.assertNotIn("NCCL_GRAPH_REGISTER", os.environ)
|
||||
|
||||
def test_disabled_decode_graph_env_is_untouched(self):
|
||||
# Without a decode graph no registration outlives the collective.
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("NCCL_GRAPH_REGISTER", None)
|
||||
self._resolve(
|
||||
cuda_graph_config=CudaGraphConfig(
|
||||
decode=PhaseConfig(backend=Backend.DISABLED),
|
||||
prefill=PhaseConfig(backend=Backend.DISABLED),
|
||||
)
|
||||
)
|
||||
self.assertNotIn("NCCL_GRAPH_REGISTER", os.environ)
|
||||
|
||||
|
||||
class TestDcpCommBackendDefault(CustomTestCase):
|
||||
|
||||
Reference in New Issue
Block a user