From 5c8fb9e9385dec105fdd8469b27b76402f6e8897 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sun, 4 Oct 2026 16:39:06 -0700 Subject: [PATCH] [Fix] Offer the fused MoE finalize all-reduce only to a block that owes the sum (#42479) --- python/sglang/srt/models/deepseek_v2.py | 2 + .../test_stage_producers_leave_sums.py | 66 ++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index e4f509c5ed..6167778f8a 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -1106,6 +1106,8 @@ class DeepseekV2MoE(nn.Module): # The mHC post-split consumes the reduced row without an RMSNorm. use_fused_finalize_all_reduce = ( self._fuse_finalize_all_reduce + # The kernel completes the tensor-parallel sum itself. + and self.reduce_results and has_shared_output and hidden_states.shape[-1] == 5120 and not self._shared_expert_tp1 diff --git a/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py b/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py index 2891822432..c31c729488 100644 --- a/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py +++ b/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py @@ -16,7 +16,9 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=5, suite="base-a-test-cpu") -# Calls that complete a TP / EP sum of a stage output in compute. +# Calls that complete a TP / EP sum of a stage output in compute. A fused +# kernel that carries the sum inside it counts as well: every MoE finalize +# all-reduce variant does. _SUMS = { "reduce_moe_output", "post_experts_all_reduce", @@ -24,6 +26,13 @@ _SUMS = { "moe_expert_parallel_all_reduce", "moe_tensor_model_parallel_all_reduce", } +_SUM_PREFIXES = ("moe_finalize_all_reduce",) + + +def _is_sum(name) -> bool: + return name in _SUMS or (name is not None and name.startswith(_SUM_PREFIXES)) + + _BUILDS_STAGES = ( "make_stages(", "declare_attn(", @@ -52,22 +61,46 @@ def _call_name(node: ast.Call): return func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", None) +def _requires(expr, names) -> bool: + """Whether ``expr`` holds only when one of ``names`` is true: the flag + itself, or an ``and`` with it as a conjunct. A negation, a comparison or an + ``or`` does not require it, so it guards nothing.""" + if isinstance(expr, ast.Attribute): + return expr.attr in names + if isinstance(expr, ast.Name): + return expr.id in names + if isinstance(expr, ast.BoolOp) and isinstance(expr.op, ast.And): + return any(_requires(value, names) for value in expr.values) + return False + + +def _reduce_results_names(tree) -> set: + """``reduce_results`` and the names bound to a condition that requires it. + A branch on such a name is guarded like a branch on the condition.""" + names = {"reduce_results"} + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and _requires(node.value, names): + names.update(t.id for t in node.targets if isinstance(t, ast.Name)) + return names + + def _unguarded_sums(source: str): """Sum calls not under ``if ... self.reduce_results ...``: such a branch is a shared class built without stage boundaries by another model. Sums inside a vocabulary-parallel embedding complete its lookup, not a stage output.""" found = [] + guards = _reduce_results_names(ast.parse(source)) def visit(node, guarded): if _is_vocab_parallel(node): return - if isinstance(node, ast.If) and "reduce_results" in ast.unparse(node.test): + if isinstance(node, ast.If) and _requires(node.test, guards): for child in node.body: visit(child, True) for child in node.orelse: visit(child, guarded) return - if isinstance(node, ast.Call) and not guarded and _call_name(node) in _SUMS: + if isinstance(node, ast.Call) and not guarded and _is_sum(_call_name(node)): found.append(f"{node.lineno}: {ast.unparse(node)}") for child in ast.iter_child_nodes(node): visit(child, guarded) @@ -90,6 +123,33 @@ class TestStageProducersLeaveSums(CustomTestCase): self.assertGreater(checked, 30) self.assertEqual(offenders, {}) + def test_fused_sums_count_and_only_a_required_flag_guards(self): + source = ( + "class Moe:\n" + " def forward(self, x):\n" + " fused = self.reduce_results and x.is_cuda\n" + " if fused:\n" + " moe_finalize_all_reduce_mhc_quant(x)\n" + " skip = not self.reduce_results\n" + " if skip:\n" + " moe_finalize_all_reduce_mhc_combine(x)\n" + " if not self.reduce_results:\n" + " tensor_model_parallel_all_reduce(x)\n" + " if self.reduce_results is False:\n" + " moe_finalize_all_reduce(x)\n" + " if x.is_cuda or self.reduce_results:\n" + " tensor_model_parallel_all_reduce(x)\n" + ) + self.assertEqual( + _unguarded_sums(source), + [ + "8: moe_finalize_all_reduce_mhc_combine(x)", + "10: tensor_model_parallel_all_reduce(x)", + "12: moe_finalize_all_reduce(x)", + "14: tensor_model_parallel_all_reduce(x)", + ], + ) + def test_guard_rejects_a_producer_that_sums_its_own_output(self): class Producer(torch.nn.Module): def __init__(self, reduce_results):