diff --git a/python/sglang/srt/arg_groups/parallel_hook.py b/python/sglang/srt/arg_groups/parallel_hook.py index f1dcda0802..ea50f71400 100644 --- a/python/sglang/srt/arg_groups/parallel_hook.py +++ b/python/sglang/srt/arg_groups/parallel_hook.py @@ -366,26 +366,37 @@ def handle_data_parallelism(server_args: Any): run_post_process_pass(server_args, _tp_lm_head_all_to_all_default) run_post_process_pass(server_args, _dp_lm_head_validation) - if resolving_view(server_args).enable_tp_lm_head_all_to_all: + if _graph_may_capture_collectives(server_args): _disable_nccl_graph_buffer_registration( - "the graph-captured TP LM-head all-to-all can deadlock with " + "graph replay of captured cross-rank collectives can hang with " "registered buffers" ) - if _graph_pool_is_pausable(server_args): - _disable_nccl_graph_buffer_registration( - "graph replay can hang once torch_memory_saver resume remaps the " - "graph pool, leaving the capture-time registrations on released pages" - ) -def _graph_pool_is_pausable(server_args: Any) -> bool: - """Whether CUDA graphs are captured into torch_memory_saver memory, which - release/resume of the `cuda_graph` tag remaps to new physical pages at the - same virtual addresses.""" - return bool( - resolving_view(server_args).enable_memory_saver - and envs.SGLANG_MEMORY_SAVER_CUDA_GRAPH.get() - ) +def _graph_may_capture_collectives(server_args: Any) -> bool: + """Whether any captured CUDA graph can hold a cross-rank collective. + + Conservative workaround, root cause still open: multi-node TP16 decode + replay hangs with NCCL_GRAPH_REGISTER on even with the TP LM-head + all-to-all and the pausable graph pool both off, and stops hanging once + the registration is off. Graph-pool address reuse is the suspected + trigger, not a confirmed mechanism, so the guard keys on any multi-rank + capture rather than on the feature flags that first exposed it. Checking + every phase also keeps the prefill-into-pausable-pool case the old pool + guard covered. + + Errs toward True. Later passes can still narrow cuda_graph_config. On the + reproducing deployment, a single 10-minute run showed no throughput + difference with the registration off; other deployment shapes are + unmeasured. + """ + view = resolving_view(server_args) + if max(view.tp_size, view.ep_size, view.dp_size) <= 1: + return False + config = view.cuda_graph_config + if config is None: + return True + return any(config[phase].backend != Backend.DISABLED for phase in Phase.ALL) def _disable_nccl_graph_buffer_registration(reason: str) -> None: diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index a43b9fc73b..2703d68c25 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -4543,14 +4543,14 @@ class TestRustServerDpAttentionControlBroadcast(unittest.TestCase): class TestTpLmHeadAllToAllNcclGraphRegister(unittest.TestCase): - """The graph-captured TP LM-head all-to-all must not run with NCCL's - graph buffer registration: registered graph-pool temporaries deadlock the - exchange under DP-rank ramps.""" + """A decode graph that can capture a collective must not run with NCCL's + graph buffer registration: the registered graph-pool temporaries alias + tensors the pool hands out later, and replay then deadlocks.""" def _resolve(self, **kwargs): # The handler reads the DP-adjusted prefill knobs the pipeline would # have settled by then; the dummy-model pipeline itself returns early. - server_args = ServerArgs( + fields = dict( model_path="dummy", attn_dp_size=2, tp_size=2, @@ -4558,8 +4558,9 @@ class TestTpLmHeadAllToAllNcclGraphRegister(unittest.TestCase): cuda_graph_config=CudaGraphConfig( prefill=PhaseConfig(backend=Backend.DISABLED) ), - **kwargs, ) + fields.update(kwargs) + server_args = ServerArgs(**fields) parallel_hook.handle_data_parallelism(server_args) return server_args @@ -4579,9 +4580,9 @@ class TestTpLmHeadAllToAllNcclGraphRegister(unittest.TestCase): self.assertEqual(os.environ["NCCL_GRAPH_REGISTER"], "1") self.assertIn("NCCL_GRAPH_REGISTER=1", "\n".join(logs.output)) - def test_without_all_to_all_env_is_untouched(self): - # Unified serving keeps the all-to-all off by default, and a decode - # node with the DP LM head never takes the all-to-all. + def test_without_all_to_all_registration_still_defaults_off(self): + # The hazard is the graph-captured collective, not the feature that + # happens to issue it, so the default holds with the all-to-all off. for kwargs in ( {}, {"disaggregation_mode": "decode", "enable_dp_lm_head": True}, @@ -4592,7 +4593,67 @@ class TestTpLmHeadAllToAllNcclGraphRegister(unittest.TestCase): self.assertFalse( resolution_result(server_args, "enable_tp_lm_head_all_to_all") ) - self.assertNotIn("NCCL_GRAPH_REGISTER", os.environ) + self.assertEqual(os.environ.get("NCCL_GRAPH_REGISTER"), "0") + + def test_plain_tp_defaults_off(self): + # The motivating shape: plain TP with the all-to-all and memory saver off. + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("NCCL_GRAPH_REGISTER", None) + server_args = self._resolve( + attn_dp_size=1, + tp_size=2, + enable_tp_lm_head_all_to_all=False, + enable_memory_saver=False, + ) + self.assertFalse( + resolution_result(server_args, "enable_tp_lm_head_all_to_all") + ) + self.assertEqual(os.environ.get("NCCL_GRAPH_REGISTER"), "0") + + def test_pausable_graph_pool_defaults_off(self): + # Isolates the torch_memory_saver trigger: the all-to-all stays off, so + # only the remapped graph pool can account for the default. + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("NCCL_GRAPH_REGISTER", None) + server_args = self._resolve(enable_memory_saver=True) + self.assertFalse( + resolution_result(server_args, "enable_tp_lm_head_all_to_all") + ) + self.assertEqual(os.environ.get("NCCL_GRAPH_REGISTER"), "0") + + def test_prefill_only_capture_defaults_off(self): + # Decode capture off, prefill still capturing into the same pausable + # pool: the collective is in a graph either way, so the registration has + # to go regardless of which phase holds it. + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("NCCL_GRAPH_REGISTER", None) + self._resolve( + enable_memory_saver=True, + cuda_graph_config=CudaGraphConfig( + decode=PhaseConfig(backend=Backend.DISABLED), + prefill=PhaseConfig(backend=Backend.FULL), + ), + ) + self.assertEqual(os.environ.get("NCCL_GRAPH_REGISTER"), "0") + + def test_single_rank_env_is_untouched(self): + # One rank issues no collective, so a capture registers nothing. + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("NCCL_GRAPH_REGISTER", None) + self._resolve(attn_dp_size=1, tp_size=1) + self.assertNotIn("NCCL_GRAPH_REGISTER", os.environ) + + def test_disabled_decode_graph_env_is_untouched(self): + # Without a decode graph no registration outlives the collective. + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("NCCL_GRAPH_REGISTER", None) + self._resolve( + cuda_graph_config=CudaGraphConfig( + decode=PhaseConfig(backend=Backend.DISABLED), + prefill=PhaseConfig(backend=Backend.DISABLED), + ) + ) + self.assertNotIn("NCCL_GRAPH_REGISTER", os.environ) class TestDcpCommBackendDefault(CustomTestCase):