From c80285ca9adc23946447128926c22e8e7dfb2b80 Mon Sep 17 00:00:00 2001 From: Aarne Talman Date: Thu, 22 Oct 2026 19:31:25 +0300 Subject: [PATCH] [Fix] [AMD] deterministic: Refuse RCCL all-reduce in deterministic mode above TP2 (#43809) Signed-off-by: Aarne Talman --- .../sglang/srt/arg_groups/attention_hook.py | 2 ++ python/sglang/srt/arg_groups/overrides.py | 18 +++++++++++ test/registered/unit/test_model_overrides.py | 32 +++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/python/sglang/srt/arg_groups/attention_hook.py b/python/sglang/srt/arg_groups/attention_hook.py index 4bf3b28178..686001ebed 100644 --- a/python/sglang/srt/arg_groups/attention_hook.py +++ b/python/sglang/srt/arg_groups/attention_hook.py @@ -14,6 +14,7 @@ from sglang.srt.arg_groups.overrides import ( _cutedsl_prefill_backend_fill, _deterministic_allreduce_fusion_disable, _deterministic_attention_backend, + _deterministic_rocm_rccl_check, _deterministic_sampling_backend, _fa4_page_constraint, _intel_xpu_page_constraint, @@ -668,6 +669,7 @@ def handle_deterministic_inference(server_args: Any): if get_platform().is_hip: # AMD: use 1-stage all-reduce kernel which is inherently deterministic # (each GPU reads all data from all GPUs, reduces locally in fixed order) + run_post_process_pass(server_args, _deterministic_rocm_rccl_check) logger.info("AMD/ROCm: Using 1-stage all-reduce kernel (deterministic)") else: # CUDA: use NCCL tree algorithm diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index 9a672420d6..3de5285c4b 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -1071,6 +1071,24 @@ def _deterministic_allreduce_fusion_disable(view: Any) -> dict: return {} +@register_post_process +def _deterministic_rocm_rccl_check(view: Any) -> dict: + # Deterministic TP on ROCm relies on the one-stage custom all-reduce; above two ranks + # RCCL's results change with the batch, even with its algorithm and protocol pinned. + if ( + view.enable_deterministic_inference + and get_platform().is_hip + and view.tp_size > 2 + and view.disable_custom_all_reduce + ): + raise ValueError( + "--disable-custom-all-reduce cannot be combined with " + "--enable-deterministic-inference on ROCm when --tp-size is above 2: " + "the all-reduce would run through RCCL, which is not batch-invariant." + ) + return {} + + @register_post_process def _deterministic_attention_backend(view: Any) -> dict: if not view.enable_deterministic_inference: diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index d36bcab75d..1d2ddedd6d 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -1829,6 +1829,38 @@ class TestGoldenModelOverrides(_IsolatedPublish): ) self.assertEqual(_deterministic_sampling_backend(view), {}) + def test_deterministic_rocm_rccl_check(self): + """Deterministic ROCm TP above 2 must not fall back to RCCL; CUDA (which + disables the custom all-reduce itself) and TP2 (one summation order) are allowed.""" + from sglang.srt.arg_groups.overrides import ( + ResolvedView, + _deterministic_rocm_rccl_check, + ) + + # (is_hip, deterministic, tp_size, disable_custom_all_reduce, raises) + for is_hip, det, tp, disable, raises in [ + (True, True, 4, True, True), + (False, True, 8, True, False), + (True, True, 2, True, False), + (True, False, 8, True, False), + (True, True, 8, False, False), + ]: + view = ResolvedView( + SimpleNamespace( + enable_deterministic_inference=det, + tp_size=tp, + disable_custom_all_reduce=disable, + ) + ) + with ( + self.subTest(is_hip=is_hip, det=det, tp=tp, disable=disable), + override_platform(is_cuda=not is_hip, is_hip=is_hip), + ): + if raises: + self.assertRaises(ValueError, _deterministic_rocm_rccl_check, view) + else: + self.assertEqual(_deterministic_rocm_rccl_check(view), {}) + def test_dllm_forces_flashinfer_with_cuda_graph(self): # CUDA path: cuda graph enabled by default -> dllm forces flashinfer. # A real dllm arch: the page pass now runs regardless of the radix