[Fix] [AMD] deterministic: Refuse RCCL all-reduce in deterministic mode above TP2 (#43809)
Signed-off-by: Aarne Talman <aarne.talman@amd.com>
This commit is contained in:
@@ -14,6 +14,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
_cutedsl_prefill_backend_fill,
|
||||
_deterministic_allreduce_fusion_disable,
|
||||
_deterministic_attention_backend,
|
||||
_deterministic_rocm_rccl_check,
|
||||
_deterministic_sampling_backend,
|
||||
_fa4_page_constraint,
|
||||
_intel_xpu_page_constraint,
|
||||
@@ -668,6 +669,7 @@ def handle_deterministic_inference(server_args: Any):
|
||||
if get_platform().is_hip:
|
||||
# AMD: use 1-stage all-reduce kernel which is inherently deterministic
|
||||
# (each GPU reads all data from all GPUs, reduces locally in fixed order)
|
||||
run_post_process_pass(server_args, _deterministic_rocm_rccl_check)
|
||||
logger.info("AMD/ROCm: Using 1-stage all-reduce kernel (deterministic)")
|
||||
else:
|
||||
# CUDA: use NCCL tree algorithm
|
||||
|
||||
@@ -1071,6 +1071,24 @@ def _deterministic_allreduce_fusion_disable(view: Any) -> dict:
|
||||
return {}
|
||||
|
||||
|
||||
@register_post_process
|
||||
def _deterministic_rocm_rccl_check(view: Any) -> dict:
|
||||
# Deterministic TP on ROCm relies on the one-stage custom all-reduce; above two ranks
|
||||
# RCCL's results change with the batch, even with its algorithm and protocol pinned.
|
||||
if (
|
||||
view.enable_deterministic_inference
|
||||
and get_platform().is_hip
|
||||
and view.tp_size > 2
|
||||
and view.disable_custom_all_reduce
|
||||
):
|
||||
raise ValueError(
|
||||
"--disable-custom-all-reduce cannot be combined with "
|
||||
"--enable-deterministic-inference on ROCm when --tp-size is above 2: "
|
||||
"the all-reduce would run through RCCL, which is not batch-invariant."
|
||||
)
|
||||
return {}
|
||||
|
||||
|
||||
@register_post_process
|
||||
def _deterministic_attention_backend(view: Any) -> dict:
|
||||
if not view.enable_deterministic_inference:
|
||||
|
||||
Reference in New Issue
Block a user