diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index b4099bfad..c0e2d284e 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -33,6 +33,7 @@ from sglang.srt.distributed import ( get_moe_tensor_parallel_world_size, get_pp_group, get_tensor_model_parallel_rank, + moe_expert_parallel_all_reduce, moe_tensor_model_parallel_all_reduce, ) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder @@ -321,6 +322,9 @@ class Qwen3MoeSparseMoeBlock(nn.Module): topk_output = self.topk(hidden_states, router_logits) final_hidden_states = self.experts(hidden_states, topk_output) + if self.ep_size > 1 and not should_allreduce_fusion: + final_hidden_states = moe_expert_parallel_all_reduce(final_hidden_states) + if ( self.tp_size > 1 and not should_allreduce_fusion diff --git a/test/registered/4-gpu-models/test_qwen3_30b.py b/test/registered/4-gpu-models/test_qwen3_30b.py index 22adb7506..073723dda 100644 --- a/test/registered/4-gpu-models/test_qwen3_30b.py +++ b/test/registered/4-gpu-models/test_qwen3_30b.py @@ -11,11 +11,8 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci( - est_time=300, - suite="stage-c-test-4-gpu-h100", - disabled="Temporarily disable the flaky test.", -) +register_cuda_ci(est_time=300, suite="stage-c-test-4-gpu-h100") + QWEN3_30B_MODEL_PATH = "Qwen/Qwen3-30B-A3B-FP8" GSM8K_BASELINE_ACCURACY = 0.85