[Fix] [AMD] deterministic: Refuse RCCL all-reduce in deterministic mode above TP2 (#43809)

Signed-off-by: Aarne Talman <aarne.talman@amd.com>
This commit is contained in:
Aarne Talman
2026-10-22 09:31:25 -07:00
committed by GitHub
parent cc811ff0c3
commit c80285ca9a
3 changed files with 52 additions and 0 deletions
@@ -14,6 +14,7 @@ from sglang.srt.arg_groups.overrides import (
_cutedsl_prefill_backend_fill,
_deterministic_allreduce_fusion_disable,
_deterministic_attention_backend,
_deterministic_rocm_rccl_check,
_deterministic_sampling_backend,
_fa4_page_constraint,
_intel_xpu_page_constraint,
@@ -668,6 +669,7 @@ def handle_deterministic_inference(server_args: Any):
if get_platform().is_hip:
# AMD: use 1-stage all-reduce kernel which is inherently deterministic
# (each GPU reads all data from all GPUs, reduces locally in fixed order)
run_post_process_pass(server_args, _deterministic_rocm_rccl_check)
logger.info("AMD/ROCm: Using 1-stage all-reduce kernel (deterministic)")
else:
# CUDA: use NCCL tree algorithm
+18
View File
@@ -1071,6 +1071,24 @@ def _deterministic_allreduce_fusion_disable(view: Any) -> dict:
return {}
@register_post_process
def _deterministic_rocm_rccl_check(view: Any) -> dict:
# Deterministic TP on ROCm relies on the one-stage custom all-reduce; above two ranks
# RCCL's results change with the batch, even with its algorithm and protocol pinned.
if (
view.enable_deterministic_inference
and get_platform().is_hip
and view.tp_size > 2
and view.disable_custom_all_reduce
):
raise ValueError(
"--disable-custom-all-reduce cannot be combined with "
"--enable-deterministic-inference on ROCm when --tp-size is above 2: "
"the all-reduce would run through RCCL, which is not batch-invariant."
)
return {}
@register_post_process
def _deterministic_attention_backend(view: Any) -> dict:
if not view.enable_deterministic_inference: