From acffb0d20d4e172981e84d1006ca10e1704f039c Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:16:44 -0700 Subject: [PATCH] [Refactor] Select reduction fusion at the consumer (#41551) --- .../srt/layers/communicator/__init__.py | 2 - .../srt/layers/communicator/boundary.py | 47 +++- .../layers/communicator/fusions/cutedsl.py | 80 +++---- .../sglang/srt/layers/communicator/layer.py | 217 ++++++++++-------- python/sglang/srt/layers/communicator/ops.py | 35 ++- .../layers/communicator/residual/access.py | 7 +- .../layers/communicator/residual/stream.py | 32 ++- python/sglang/srt/models/bailing_moe.py | 5 +- .../sglang/srt/models/bailing_moe_linear.py | 5 +- python/sglang/srt/models/bailing_moe_v3.py | 5 +- python/sglang/srt/models/deepseek_v2.py | 8 +- python/sglang/srt/models/gigachat35.py | 7 +- python/sglang/srt/models/glm4_moe.py | 5 +- python/sglang/srt/models/glm4_moe_lite.py | 5 +- python/sglang/srt/models/glm5_next.py | 5 +- python/sglang/srt/models/gpt_oss.py | 5 +- python/sglang/srt/models/laguna.py | 5 +- python/sglang/srt/models/llada2.py | 7 +- python/sglang/srt/models/mimo_v2.py | 5 +- python/sglang/srt/models/minimax_m2.py | 5 +- python/sglang/srt/models/minimax_m3.py | 5 +- python/sglang/srt/models/nemotron_h.py | 5 +- python/sglang/srt/models/qwen2_moe.py | 5 +- python/sglang/srt/models/qwen3_5.py | 3 +- python/sglang/srt/models/qwen3_vl.py | 7 +- python/sglang/srt/models/qwen3_vl_moe.py | 5 +- python/sglang/srt/models/sarvam_moe.py | 5 +- python/sglang/srt/models/sdar.py | 7 +- python/sglang/srt/models/sdar_moe.py | 5 +- python/sglang/srt/models/step3p5.py | 5 +- python/sglang/srt/models/xllm.py | 7 +- .../ops/test_aiter_allreduce_fusion_amd.py | 156 +++++-------- .../unit/layers/moe/test_cutedsl_ar_fusion.py | 133 +++++++---- .../unit/layers/test_batch_owned_residual.py | 3 + .../unit/layers/test_communicator_ffn_exit.py | 46 +--- .../layers/test_declared_decoder_boundary.py | 4 +- .../test_layer_communicator_fusion_gate.py | 26 ++- .../unit/layers/test_residual_stream.py | 80 ++++++- .../lora/test_triton_dp_attention_unit.py | 2 + .../test_aux_capture_deferred_allreduce.py | 9 +- .../unit/models/test_layer_stack_exit.py | 5 +- .../models/test_nemotron_h_mtp_reduction.py | 7 +- .../unit/models/test_nemotron_h_stages.py | 11 +- .../test_step3p5_dense_reduce_scatter.py | 2 +- 44 files changed, 639 insertions(+), 396 deletions(-) diff --git a/python/sglang/srt/layers/communicator/__init__.py b/python/sglang/srt/layers/communicator/__init__.py index cfae0136d1..6eddee083f 100644 --- a/python/sglang/srt/layers/communicator/__init__.py +++ b/python/sglang/srt/layers/communicator/__init__.py @@ -40,7 +40,6 @@ from sglang.srt.layers.communicator.boundary import ( from sglang.srt.layers.communicator.layer import ( FfnCompletion, FfnExit, - FfnExitFusion, LayerCommunicator, LayerFacts, MHCLayerCommunicator, @@ -91,7 +90,6 @@ __all__ = [ "FUSE_ALLREDUCE_MAX_BATCH_SIZE", "FfnCompletion", "FfnExit", - "FfnExitFusion", "FusedMlpInput", "HandoffOutput", "LayerCommunicator", diff --git a/python/sglang/srt/layers/communicator/boundary.py b/python/sglang/srt/layers/communicator/boundary.py index 6567240804..f866fce27e 100644 --- a/python/sglang/srt/layers/communicator/boundary.py +++ b/python/sglang/srt/layers/communicator/boundary.py @@ -49,6 +49,7 @@ from sglang.srt.layers.communicator.ops import ( _read_input, move_rows, tp_reduce_scatter, + tp_slice, ) from sglang.srt.layers.communicator.residual import ( LayerResidual, @@ -551,6 +552,7 @@ class StageEntry(msgspec.Struct, frozen=True): capture_move: Optional[Callable] = None capture_move_allocates: bool = False capture_preserves_residual: Optional[Callable] = None + input_sum: Optional[SumGroup] = None class BoundarySteps(msgspec.Struct, frozen=True): @@ -724,6 +726,20 @@ def _bind_consumer( cp_moves=cp_moves, enters_stack=enters_stack, ) + completed_step = None + if edge.produced.always_leaves: + completed_step, _, _ = _select_input_steps( + msgspec.structs.replace(edge.produced, always_leaves=False), + residual=edge.residual, + residual_to=edge.residual_to, + need=edge.need, + adds_plainly=adds_plainly, + fusions=(), + force_layernorm_before_gather=force_layernorm_before_gather, + residual_joins_sum=False, + cp_moves=cp_moves, + enters_stack=enters_stack, + ) # A written stream entering the first physical layer must not run enter # again (e.g. MHC expansion). Both alternatives are bound at construction. written_step = None @@ -747,7 +763,8 @@ def _bind_consumer( step=step, adds_plainly=adds_plainly, carried_fusions=carried_fusions if plain else (), - owes_by_construction=edge.produced.always_leaves, + expected_sum=edge.produced.group if edge.produced.always_leaves else None, + completed_step=completed_step, written_step=written_step, ), input_move=input_move, @@ -826,12 +843,18 @@ def _select_boundary_steps( handoff=attention_handoff, capture_move=into_attention.capture_move, capture_move_allocates=into_attention.capture_move_allocates, + input_sum=edges.into_attention.produced.group + if edges.into_attention.produced.always_leaves + else None, ), ffn=StageEntry( prepare=into_ffn.prepare, input_rows=into_ffn.input_rows, input_move=into_ffn.input_move, fused=into_ffn.fused, + input_sum=edges.into_ffn.produced.group + if edges.into_ffn.produced.always_leaves + else None, ), ffn_output=edges.out_of_ffn.produced, ffn_output_move=out_of_ffn.output_move, @@ -895,17 +918,19 @@ def _select_input_steps( if residual == residual_to: # The residual stays on each rank's slice while the stage takes the # rows around it (MHC on an input-scattered batch). - if gathered or owes is not SumGroup.ATTN_TP: + if gathered or owes not in (None, SumGroup.ATTN_TP): raise NotImplementedError(f"{produced=} {residual=} {need=}") return ( - partial(_mlp_input_on_residual_shard, read=read), + partial( + _mlp_input_on_residual_shard, read=read, reduces=owes is not None + ), (), None, ) # A reduce-scatter completes the TP sum onto each rank's slice, which # the stage takes: its group is the TP group without attention DP or CP. if ( - owes is not SumGroup.TP + owes not in (None, SumGroup.TP) or residual.sharded or TokenAxis.ATTN_TP_SCATTER not in need.gathers_itself or residual_to.sharded != {TokenAxis.ATTN_TP_SCATTER} @@ -914,7 +939,7 @@ def _select_input_steps( return ( partial( _read_input, - layer_input=tp_reduce_scatter, + layer_input=tp_reduce_scatter if owes is not None else tp_slice, enters_stack=enters_stack, read=read, ), @@ -974,7 +999,17 @@ def _select_input_steps( if not gathered: if owes is None: if gathers_residual: - raise NotImplementedError(f"{produced=} {residual=} {need=}") + return ( + partial( + _mlp_input_without_dp, + gathers_residual=True, + fusions=(), + group=None, + read=read, + ), + (), + None, + ) return ( partial( _read_input, diff --git a/python/sglang/srt/layers/communicator/fusions/cutedsl.py b/python/sglang/srt/layers/communicator/fusions/cutedsl.py index 75d7798de5..ee6d1992fa 100644 --- a/python/sglang/srt/layers/communicator/fusions/cutedsl.py +++ b/python/sglang/srt/layers/communicator/fusions/cutedsl.py @@ -14,7 +14,6 @@ from typing import Callable, Optional, Sequence import torch from sglang.srt.layers.communicator import ( - FfnExitFusion, FusedMlpInput, HandoffOutput, LayerCommunicator, @@ -201,33 +200,27 @@ class CuteDSLFusion: construction. At the attention input: the MoE finalize + AR + add + norm of a handoff the previous layer left, and the AR + add + norm of a sum it left. At the FFN input: the attention output's AR + add + norm. At the FFN exit: - handing the MoE finalize, or the all-reduce, to the next layer's kernel. - install_cutedsl_fusion gives it the model's service and chooses the exit - from the layer after this one; until then no kernel takes a batch.""" + handing off a deferred MoE finalize with its unfused completion. + install_cutedsl_fusion supplies the service and producer capability; + consumer eligibility is checked only where the output is consumed.""" def __init__(self) -> None: self.service: CuteDSLFusionService | None = None - # The FFN exit's choices, from what install() knows of the next layer. + # Producer capabilities; consumer kernel selection is independent. self.hands_off_finalize = False - self.next_input_absorbs = False + self.requires_local_reduction = False def install( self, service: CuteDSLFusionService, *, hands_off_finalize: bool, - next_input_absorbs: bool, output_is_replicated: bool, ) -> None: - """Take the model's fusion service and choose the FFN exit's kernels: - the MoE may hand off its finalize when it can and something after it - takes the handoff (``hands_off_finalize``); the FFN may leave its - all-reduce to the next layer's AR + norm when that layer has one, unless - a replicated output follows the reduction (moving it would scale that - by tp).""" + """Install the shared workspace and this producer's capabilities.""" self.service = service self.hands_off_finalize = hands_off_finalize - self.next_input_absorbs = next_input_absorbs and not output_is_replicated + self.requires_local_reduction = output_is_replicated def attention_input(self, layer: LayerCommunicator) -> tuple: return ( @@ -254,12 +247,6 @@ class CuteDSLFusion: ) return () - def ffn_exit(self, layer: LayerCommunicator) -> tuple: - return ( - partial(self._defer_moe_finalize, layer), - partial(self._absorb_all_reduce, layer), - ) - def _finalize_output_and_update_and_read_residual( self, layer, owed, residual, forward_batch, post_residual_addition ): @@ -330,26 +317,28 @@ class CuteDSLFusion: and not get_exec().comm.enable_quant_communications ) - def _defer_moe_finalize( - self, layer, forward_batch: ForwardBatch - ) -> Optional[FfnExitFusion]: - """The MoE hands its unfinalized output to the next layer's finalize + - all-reduce + norm.""" - if self.hands_off_finalize and self._should_use_finalize( - layer, forward_batch, int(forward_batch.input_ids.shape[0]) - ): - return FfnExitFusion.DEFER_MOE_FINALIZE - return None + def can_defer_all_reduce(self, layer, forward_batch: ForwardBatch) -> bool: + """Preserve the producer's fused sum path when this workspace is usable. - def _absorb_all_reduce( - self, layer, forward_batch: ForwardBatch - ) -> Optional[FfnExitFusion]: - """The next layer's AR + norm takes the post-experts all-reduce.""" - if self.next_input_absorbs and self._can_consume_post_moe_all_reduce( - layer, forward_batch, int(forward_batch.input_ids.shape[0]) - ): - return FfnExitFusion.NEXT_INPUT - return None + The next consumer still selects its kernel and completes the sum with + an ordinary all-reduce if that kernel declines the actual input. + """ + return ( + self._common_eligible( + layer, forward_batch, int(forward_batch.input_ids.shape[0]) + ) + and not get_exec().comm.enable_quant_communications + ) + + def can_defer_finalize(self, layer, forward_batch: ForwardBatch) -> bool: + """Whether this producer can emit a handoff with an unfused fallback.""" + return ( + self.hands_off_finalize + and not self.requires_local_reduction + and self._should_use_finalize( + layer, forward_batch, int(forward_batch.input_ids.shape[0]) + ) + ) def _common_eligible(self, layer, forward_batch: ForwardBatch, m: int) -> bool: parallel = get_parallel() @@ -383,7 +372,6 @@ def install_cutedsl_fusion( rms_epsilon: float, can_defer_finalize: _LayerPredicate, requires_local_reduction: _LayerPredicate | None = None, - final_norm_consumes_handoff: bool = False, label: str, ) -> CuteDSLFusionService | None: """One shared workspace handle per fusion-enabled layer, or None. @@ -419,23 +407,15 @@ def install_cutedsl_fusion( rms_epsilon=rms_epsilon, ) hands_off = 0 - for index, layer in enumerate(layers): + for layer in layers: fusion = _fusion_of(layer) if fusion is None: continue - successor = layers[index + 1] if index + 1 < len(layers) else None - if successor is None: - has_consumer = final_norm_consumes_handoff - else: - has_consumer = _fusion_of(successor) is not None - hands_off_finalize = bool(can_defer_finalize(layer)) and has_consumer + hands_off_finalize = bool(can_defer_finalize(layer)) hands_off += hands_off_finalize fusion.install( service, hands_off_finalize=hands_off_finalize, - # False on the last layer: the final norm does not all-reduce. - next_input_absorbs=successor is not None - and _fusion_of(successor) is not None, output_is_replicated=requires_local_reduction is not None and bool(requires_local_reduction(layer)), ) diff --git a/python/sglang/srt/layers/communicator/layer.py b/python/sglang/srt/layers/communicator/layer.py index 179128e5df..279eb29a18 100644 --- a/python/sglang/srt/layers/communicator/layer.py +++ b/python/sglang/srt/layers/communicator/layer.py @@ -14,7 +14,6 @@ """The communicator a model layer uses.""" from dataclasses import dataclass -from enum import Enum, auto from functools import cached_property, partial from typing import Callable, Optional, Protocol, Tuple, Union @@ -49,7 +48,6 @@ from sglang.srt.layers.communicator.layout import ( SumGroup, TokenAxis, _batch_shards_over_cp, - _batch_size, _ffn_has_tokens, _gathers_over_attention_cp, _generic_prefill_cp_shards_tokens, @@ -86,13 +84,13 @@ from sglang.srt.layers.communicator.residual.access import ( written, ) from sglang.srt.layers.communicator.residual.add_norm import ( + ADD, PLAIN_RESIDUAL, - aiter_all_reduce_fusion_enabled_for, apply_aiter_all_reduce_fusion, apply_flashinfer_allreduce_fusion, ) from sglang.srt.layers.communicator.residual.mhc import MHCState -from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers.communicator.residual.stream import DeclaredSum, ResidualStream from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, is_enable_moe_cp_allgather, @@ -156,11 +154,39 @@ class LayerFacts: ) -def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool: - """Whether the next layer's all-reduce is the one the FFN would have run - itself, as the MoE declares it (post_experts_sum_is_one_all_reduce), on a - batch the FFN runs.""" - return _ffn_has_tokens(forward_batch) and post_experts_sum_is_one_all_reduce() +def _can_defer_ffn_reduction(forward_batch: ForwardBatch, boundary=None) -> bool: + """Admit ordinary single-sum outputs and the existing fused LoRA path. + + A fused-kernel fallback completes the partial-output contract selected + before the producer ran, without recomputing the producer's LoRA path. + """ + if not _ffn_has_tokens(forward_batch): + return False + if post_experts_sum_is_one_all_reduce(): + return True + # LoRA-B is replicated and linear. TP1 shared experts add on rank zero + # when the sum is deferred. Preserve these fused paths, but keep their + # producer-side reduction order when no fused consumer is enabled. + if not ( + (get_lora().enable_lora or envs.SGLANG_SHARED_EXPERT_TP1.get()) + and get_moe_a2a_backend().is_none() + and not get_exec().comm.enable_quant_communications + and post_experts_reduction_group() is get_parallel().tp_group + ): + return False + if ( + boundary is not None + and boundary.fusions is not None + and boundary.fusions.can_defer_all_reduce(boundary, forward_batch) + ): + return True + if apply_flashinfer_allreduce_fusion(forward_batch.input_ids.shape[0]): + return True + # Aiter also checks width and bytes, so use the actual residual storage. + residual = forward_batch.residual_stream.residual + return residual is not None and apply_aiter_all_reduce_fusion( + residual, forward_batch + ) class StageCommunicator: @@ -205,12 +231,20 @@ class StageCommunicator: stream = residual_batch.current(forward_batch) if residual is not stream: raise RuntimeError("residual alias belongs to a different invocation") - if stream is not None: - if stream.pending is not None: - call["update"] = stream.pending.update - if stream.pending is None and stream.residual is not None: - call["written"] = True - hidden_states, residual = stream.input(hidden_states) + if ( + stream.pending is None + and stream.residual is None + and entry.input_sum is not None + ): + hidden_states = stream.leave( + hidden_states, ADD, declared_sum=entry.input_sum + ) + if stream.pending is not None: + call["update"] = stream.pending.update + call["pending"] = stream.pending + if stream.pending is None and stream.residual is not None: + call["written"] = True + hidden_states, residual = stream.input(hidden_states) hidden_states, residual = entry.prepare( hidden_states, residual, forward_batch, self.norm, context, **call ) @@ -239,9 +273,15 @@ class LayerFusions(Protocol): def ffn_input(self, layer: "LayerCommunicator") -> Tuple["FusedMlpInput", ...]: ... - def ffn_exit( - self, layer: "LayerCommunicator" - ) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]: ... + requires_local_reduction: bool + + def can_defer_all_reduce( + self, layer: "LayerCommunicator", forward_batch: ForwardBatch + ) -> bool: ... + + def can_defer_finalize( + self, layer: "LayerCommunicator", forward_batch: ForwardBatch + ) -> bool: ... class LayerCommunicator: @@ -255,7 +295,6 @@ class LayerCommunicator: # Communicators built without __init__ (e.g. test doubles) publish no LoRA # layout and try no fused kernel at the FFN exit. _publish_lora_layout: bool = False - _ffn_exit_fusions: Tuple[Callable, ...] = () # A plain residual unless the layer is built with its own. _residual: LayerResidual = PLAIN_RESIDUAL # The fused kernels a backend gives the layer, if any. @@ -303,8 +342,6 @@ class LayerCommunicator: ) # The fused kernels every batch's attention input tries first. self._attn_input_fusions = self._select_attn_input_fusions() - # The fused kernels of the next layer's input the FFN exit tries first. - self._ffn_exit_fusions = self._select_ffn_exit_fusions() self._speculative_algo = SpeculativeAlgorithm.from_string( get_spec().speculative_algorithm ) @@ -433,6 +470,9 @@ class LayerCommunicator: handoff=None if is_ffn else _hand_qkv_hook_its_input, fused=into.fused, capture_move=into.capture_move, + input_sum=into_edge.produced.group + if into_edge.produced.always_leaves + else None, ) self._steps = BoundarySteps( attention=None if is_ffn else entry, @@ -641,7 +681,11 @@ class LayerCommunicator: def snapshot(self, hidden_states, residual, *, at_input: bool = False): group = None - if residual is not None and self.stage_edges is not None: + if ( + residual is not None + and not isinstance(residual, ResidualStream) + and self.stage_edges is not None + ): edge = self.stage_edges[0 if at_input else 1] if edge.produced.always_leaves: group = _sum_group(edge.produced.group) @@ -663,7 +707,12 @@ class LayerCommunicator: if storage is not None and storage.shape[0] == 0: return hidden_states, None hidden_states = ( - residual.complete(hidden_states) + ( + hidden_states + if residual.pending is not None + and isinstance(residual.pending.owed, DeclaredSum) + else residual.complete(hidden_states) + ) if isinstance(residual, ResidualStream) else reduce_output(hidden_states) ) @@ -675,8 +724,10 @@ class LayerCommunicator: residual_in_hidden=self._residual.ffn_update.at_producer, allow_missing_residual=allow_missing_residual, ) + steps = self._batch_steps(forward_batch) + entry = steps.attention or steps.ffn hidden_states, stream = ResidualStream.arrive( - hidden_states, residual, self._residual.ffn_update + hidden_states, residual, ADD, declared_sum=entry.input_sum ) forward_batch.residual_stream = stream return hidden_states, stream @@ -934,7 +985,9 @@ class LayerCommunicator: if isinstance(residual, ResidualStream) and self.stage_edges is None: # The paired boundary records the attention contribution once. hidden_states = residual.leave( - hidden_states, self._residual.attention_update + hidden_states, + self._residual.attention_update, + declared_sum=steps.ffn.input_sum, ) return self.ffn.prepare( hidden_states, @@ -969,16 +1022,34 @@ class LayerCommunicator: residual, stream, self._batch_steps(forward_batch).ffn_output.update, + self._declared_ffn_sum( + self._batch_steps(forward_batch), + self._ffn_leaves_sum_to_reduce_scatter( + forward_batch, self._postprocess_dp_step(forward_batch) + ), + ), ) @staticmethod - def _leave_ffn_output(hidden_states, residual, stream, update): + def _leave_ffn_output(hidden_states, residual, stream, update, declared_sum=None): if stream is None: return hidden_states, residual stream.residual = residual if update.at_producer: return stream.write(hidden_states), stream - return stream.leave(hidden_states, update), stream + return stream.leave(hidden_states, update, declared_sum=declared_sum), stream + + @staticmethod + def _declared_ffn_sum(steps, skipped_reduction): + # The input-scattered path leaves the sum for the next input's TP + # reduce-scatter. Other skip paths complete it in the output move. + if ( + skipped_reduction + and not steps.returns_over_dp + and not steps.ffn_output_move_completes_sum + ): + return SumGroup.TP + return None def _local_token_move_can_go_to_next_layer( self, forward_batch: ForwardBatch @@ -1042,14 +1113,13 @@ class LayerCommunicator: mlp_reduce_scatter=mlp_reduce_scatter, complete=complete_now, ) - fusion = next( - filter(None, (fused(forward_batch) for fused in self._ffn_exit_fusions)), - None, + defer_moe_finalize = ( + self.fusions is not None + and self.fusions.can_defer_finalize(self, forward_batch) ) - defer_moe_finalize = fusion is FfnExitFusion.DEFER_MOE_FINALIZE - # A fused kernel that takes the sum, a handoff included, skips the - # post-experts all-reduce. - fuse_mlp_allreduce = fusion is not None or self._ffn_sum_moves_to_next_layer( + # Producers declare remaining work independently of the kernel chosen + # by the consumer. Every handoff also carries an unfused completion. + fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer( forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step ) if fuse_mlp_allreduce: @@ -1205,28 +1275,12 @@ class LayerCommunicator: return stream.leave(output, update), stream return output, residual - def _select_ffn_exit_fusions( - self, - ) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]: - """The fused kernels of the next layer's input that may take this layer's - FFN sum, in the order they are tried. Each returns what the exit does - when its kernel takes this batch, or None. A backend's come first.""" - given = self.fusions.ffn_exit(self) if self.fusions else () - return (*given, self._next_input_norm_takes_ffn_sum) - - def _next_input_norm_takes_ffn_sum( - self, forward_batch: ForwardBatch - ) -> Optional["FfnExitFusion"]: - """The aiter / flashinfer AR + add + norm of the next layer's input.""" - if self.should_fuse_mlp_allreduce_with_next_layer(forward_batch): - return FfnExitFusion.NEXT_INPUT - return None - def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool: # Under the MoE-CP all-gather the fusion path would skip postprocess_layer # and its MoE-CP scatter, leaving hidden_states longer than the residual. if ( - is_enable_moe_cp_allgather() + (self.fusions is not None and self.fusions.requires_local_reduction) + or is_enable_moe_cp_allgather() or not self._batch_steps(forward_batch).ffn_sum_is_movable ): return False @@ -1252,29 +1306,6 @@ class LayerCommunicator: return not get_attn_tp_context().input_scattered - # NOTE: This function will cause torch recompilation - def should_fuse_mlp_allreduce_with_next_layer( - self, forward_batch: ForwardBatch - ) -> bool: - if not self._ffn_sum_can_move_to_next_layer(forward_batch): - return False - batch_size = _batch_size(forward_batch) - return ( - ( - apply_flashinfer_allreduce_fusion(batch_size) - or ( - _use_aiter - and batch_size > 0 - and get_parallel().tp_size != 6 - and not is_dp_attention_enabled() - and get_moe_a2a_backend().is_none() - and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode) - ) - ) - and (not self.is_last_layer) - and (self._context.tp_size > 1) - ) - def _ffn_sum_moves_to_next_layer( self, forward_batch: ForwardBatch, @@ -1289,7 +1320,7 @@ class LayerCommunicator: self._context.tp_size > 1 and not self.is_last_layer and self._ffn_sum_can_move_to_next_layer(forward_batch) - and _unfused_completion_matches_the_ffn(forward_batch) + and _can_defer_ffn_reduction(forward_batch, self) and not mlp_reduce_scatter # Under attention DP the next layer must also run postprocess's # scatter back to this rank's tokens, and nothing more. @@ -1303,16 +1334,6 @@ class LayerCommunicator: ) -class FfnExitFusion(Enum): - """What an FFN exit does when a fused kernel of the next layer's input takes - its sum.""" - - # The MoE hands its unfinalized output on; the next input finalizes it. - DEFER_MOE_FINALIZE = auto() - # The FFN leaves its all-reduce to the next input norm. - NEXT_INPUT = auto() - - class FfnCompletion(msgspec.Struct, frozen=True): """One FFN's reduction decision. The flags are published while the FFN runs; ``complete(hidden_states, residual)`` then either wraps the output for the @@ -1337,17 +1358,20 @@ class MixerExit: ``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its row-parallel output projection to skip the all-reduce. It skips when the stage's output always leaves its sum (to an FFN stage, which completes it in - its input), and when it may leave it and the fused kernel takes it into the - next input norm.""" + its input), and when its output declaration permits deferring the sum. The + consumer decides whether to fuse its completion with the input norm.""" - __slots__ = ("skips_reduction", "_hands_on", "_scope", "_update") + __slots__ = ("skips_reduction", "_hands_on", "_scope", "_update", "_declared_sum") def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch): produced = communicator._batch_steps(forward_batch).ffn_output self._update = produced.update + self._declared_sum = produced.group if produced.always_leaves else None self._hands_on = ( produced.leaves_for_next_layer - and communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch) + and communicator._context.tp_size > 1 + and not is_dp_attention_enabled() + and communicator._ffn_sum_can_move_to_next_layer(forward_batch) ) self.skips_reduction = produced.always_leaves or self._hands_on self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction) @@ -1367,7 +1391,9 @@ class MixerExit: hidden_states, group=get_parallel().tp_group ) if isinstance(residual, ResidualStream): - return residual.leave(hidden_states, self._update), residual + return residual.leave( + hidden_states, self._update, declared_sum=self._declared_sum + ), residual return hidden_states @@ -1384,6 +1410,7 @@ class FfnExit: "mlp_reduce_scatter", "_complete", "_update", + "_declared_sum", "_scope", ) @@ -1395,7 +1422,11 @@ class FfnExit: self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce self.mlp_reduce_scatter = completion.mlp_reduce_scatter self._complete = completion.complete - self._update = communicator._batch_steps(forward_batch).ffn_output.update + steps = communicator._batch_steps(forward_batch) + self._update = steps.ffn_output.update + self._declared_sum = communicator._declared_ffn_sum( + steps, completion.mlp_reduce_scatter + ) self._scope = get_forward().scoped( fuse_mlp_allreduce=self.fuse_mlp_allreduce, mlp_reduce_scatter=self.mlp_reduce_scatter, @@ -1422,7 +1453,7 @@ class FfnExit: else: hidden_states, residual = self._complete(hidden_states, residual) return self.communicator._leave_ffn_output( - hidden_states, residual, stream, self._update + hidden_states, residual, stream, self._update, self._declared_sum ) diff --git a/python/sglang/srt/layers/communicator/ops.py b/python/sglang/srt/layers/communicator/ops.py index 3f1e62e009..e42c526fb2 100644 --- a/python/sglang/srt/layers/communicator/ops.py +++ b/python/sglang/srt/layers/communicator/ops.py @@ -52,6 +52,7 @@ from sglang.srt.layers.communicator.output import ( ) from sglang.srt.layers.communicator.residual import StageRead, StageUpdate from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ +from sglang.srt.layers.communicator.residual.stream import DeclaredSum from sglang.srt.layers.dp_attention import ( attn_tp_all_gather_into_tensor, attn_tp_reduce_scatter_tensor, @@ -102,6 +103,14 @@ def tp_reduce_scatter( return output, residual +def tp_slice(hidden_states, residual, context): + """The rows reduce-scatter would return, when the sum is already complete.""" + hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank].clone() + if residual is not None: + residual = residual.tensor_split(context.tp_size)[context.tp_rank] + return hidden_states, residual + + class CommunicateSimpleFn: @staticmethod def _trivial( @@ -263,7 +272,7 @@ def _mlp_input_without_dp( hidden_states = _mlp_input_reduce_output( hidden_states, forward_batch, may_quantize=update.adds_plainly ) - else: + elif group is SumGroup.TP: hidden_states = tensor_model_parallel_all_reduce(hidden_states) if _is_npu and context.cache is not None: _ = prepare_weight_cache(hidden_states, context.cache) @@ -411,7 +420,9 @@ def _consumer_step( step: Callable, adds_plainly: bool, carried_fusions: Tuple[Callable, ...], - owes_by_construction: bool = False, + expected_sum: Optional[SumGroup] = None, + completed_step: Optional[Callable] = None, + pending=None, written_step: Optional[Callable] = None, written: bool = False, update: StageUpdate = ADD, @@ -422,18 +433,26 @@ def _consumer_step( complete value. Under attention DP the reduction back to this rank's tokens comes first; otherwise one of ``carried_fusions`` may complete it with the residual add and the read, and an empty batch has nothing to sum. An input - that owes its sum by construction (``owes_by_construction``) is that sum's - only carrier. ``call`` is what the stage's read takes (the attention's + whose declared sum is already completed uses ``completed_step`` instead; + a pending declared sum must match ``expected_sum``. ``call`` is what the stage's read takes (the attention's ``quant_format`` and ``post_residual_addition``).""" if residual is not None and update.adds_plainly != adds_plainly: raise RuntimeError("producer update does not match the boundary's capability") + if pending is not None: + if isinstance(pending.owed, DeclaredSum): + if pending.owed.group is not expected_sum: + raise RuntimeError("producer sum does not match the input declaration") + elif expected_sum is not None: + if pending.owed is not None: + raise RuntimeError("a declared input sum carried another completion") + step = completed_step if written and written_step is not None: step = written_step if not isinstance(hidden_states, torch.Tensor): owed = hidden_states if residual is None: raise RuntimeError(f"{type(owed).__name__} requires residual input") - if owes_by_construction: + if expected_sum is not None: raise RuntimeError( f"an input that owes its sum by construction arrived as " f"{type(owed).__name__}" @@ -538,6 +557,7 @@ def _mlp_input_on_residual_shard( layernorm: torch.nn.Module, context: CommunicateContext, *, + reduces: bool = True, read: StageRead, update: StageUpdate, ): @@ -548,7 +568,10 @@ def _mlp_input_on_residual_shard( if hidden_states.shape[0] == 0: return hidden_states, hidden_states shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank] - get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states) + if reduces: + get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states) + else: + shard = shard.clone() shard, residual = read.update_and_read(update, shard, residual, layernorm) attn_tp_all_gather_into_tensor(hidden_states, shard) return hidden_states, residual diff --git a/python/sglang/srt/layers/communicator/residual/access.py b/python/sglang/srt/layers/communicator/residual/access.py index a3846c085d..bd70243450 100644 --- a/python/sglang/srt/layers/communicator/residual/access.py +++ b/python/sglang/srt/layers/communicator/residual/access.py @@ -82,6 +82,7 @@ def finish_layer_stack( forward_batch: ForwardBatch, *, final_norm_takes_handoff: bool = False, + preserve_declared: bool = False, ) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]: """Complete what this layer left for a next layer. Call it on the last layer of this rank before its output reaches the final norm, the next @@ -92,7 +93,11 @@ def finish_layer_stack( stream = forward_batch.residual_stream if residual is not stream: raise RuntimeError("residual alias belongs to a different invocation") - output = stream.finish(hidden_states, takes_handoff=final_norm_takes_handoff) + output = stream.finish( + hidden_states, + takes_handoff=final_norm_takes_handoff, + preserve_declared=preserve_declared, + ) forward_batch.residual_stream = None return output if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput): diff --git a/python/sglang/srt/layers/communicator/residual/stream.py b/python/sglang/srt/layers/communicator/residual/stream.py index da7936eaa1..a95da546c5 100644 --- a/python/sglang/srt/layers/communicator/residual/stream.py +++ b/python/sglang/srt/layers/communicator/residual/stream.py @@ -19,6 +19,7 @@ import msgspec import torch from sglang.srt.distributed import GroupCoordinator +from sglang.srt.layers.communicator.layout import SumGroup, _sum_group from sglang.srt.layers.communicator.output import HandoffOutput, UnreducedOutput from sglang.srt.layers.communicator.residual import StageUpdate @@ -35,12 +36,21 @@ class CarriedSum(msgspec.Struct, frozen=True): return self.group.all_reduce(value) +class DeclaredSum(msgspec.Struct, frozen=True): + """A sum every output of this producer owes to its declared input edge.""" + + group: SumGroup + + def complete(self, value): + return _sum_group(self.group).all_reduce(value) + + class Contribution(msgspec.Struct): """The sole owner of an output's value, update and remaining work.""" value: Optional[torch.Tensor] update: StageUpdate - owed: Union[CarriedSum, HandoffOutput, None] = None + owed: Union[CarriedSum, DeclaredSum, HandoffOutput, None] = None def for_boundary(self): # These forms are private inputs to the existing fused-kernel adapters. @@ -53,7 +63,7 @@ class Contribution(msgspec.Struct): return self.value def complete(self): - if isinstance(self.owed, CarriedSum): + if isinstance(self.owed, (CarriedSum, DeclaredSum)): value = self.owed.complete(self.value) elif isinstance(self.owed, HandoffOutput): value = self.owed.complete() @@ -84,22 +94,26 @@ class ResidualStream: self.pending = None @classmethod - def arrive(cls, hidden, residual, update): + def arrive(cls, hidden, residual, update, *, declared_sum=None): """Rebuild an explicit tensor handoff, including each TBO microbatch.""" stream = cls(residual) if residual is None: return stream.write(hidden), stream - return stream.leave(hidden, update), stream + return stream.leave(hidden, update, declared_sum=declared_sum), stream def write(self, residual): self.residual = residual self.pending = None return residual - def leave(self, output, update): + def leave(self, output, update, *, declared_sum=None): if self.pending is not None: raise RuntimeError("cannot replace an unconsumed producer contribution") - if isinstance(output, UnreducedOutput): + if declared_sum is not None: + if not isinstance(output, torch.Tensor): + raise RuntimeError("a declared sum cannot carry another completion") + contribution = Contribution(output, update, DeclaredSum(declared_sum)) + elif isinstance(output, UnreducedOutput): contribution = Contribution( output.partial, update, @@ -151,7 +165,7 @@ class ResidualStream: return hidden.clone() if pending.owed is None: value = pending.value - elif isinstance(pending.owed, CarriedSum): + elif isinstance(pending.owed, (CarriedSum, DeclaredSum)): value = pending.owed.complete(pending.value.clone()) else: raise NotImplementedError("a finalize handoff requires main-output capture") @@ -159,10 +173,12 @@ class ResidualStream: value = group.all_reduce(value.clone()) return value.clone() if self.residual is None else value + self.residual - def finish(self, hidden, *, takes_handoff=False): + def finish(self, hidden, *, takes_handoff=False, preserve_declared=False): self.check(hidden) if self.pending is None: return hidden, None + if preserve_declared and isinstance(self.pending.owed, DeclaredSum): + return self.pending.value, self.residual if takes_handoff and isinstance(self.pending.owed, HandoffOutput): return self.pending.owed, self.residual return self.pending.complete(), self.residual diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index beb608a05b..914189bb24 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -848,7 +848,10 @@ class BailingMoEModel(nn.Module): ) last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index 464b6c9b8e..e1836d5992 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -1008,7 +1008,10 @@ class BailingMoELinearModel(nn.Module): ) last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index d8b8835a08..904194d4f4 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -1401,7 +1401,10 @@ class BailingMoELinearModel(nn.Module): last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index e109c6a915..218c087e8e 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -2979,7 +2979,7 @@ class DeepseekV2Model(nn.Module): install_cutedsl_fusion( # PP pads self.layers with PPMissingLayer, which has no communicator. - # A stage's last layer has no successor, so it keeps its reduction. + # A pipeline exit completes any handoff before sending its tensors. self.layers[self.start_layer : self.end_layer], hidden_size=config.hidden_size, top_k=config.num_experts_per_tok, @@ -2999,7 +2999,6 @@ class DeepseekV2Model(nn.Module): if isinstance(layer.mlp, DeepseekV2MoE) else layer.mlp.tp_size == 1 ), - final_norm_consumes_handoff=False, label="DeepSeek-V3/GLM", ) @@ -3135,7 +3134,10 @@ class DeepseekV2Model(nn.Module): last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: proxy_tensors = { diff --git a/python/sglang/srt/models/gigachat35.py b/python/sglang/srt/models/gigachat35.py index 8f88179448..3147aa42f7 100644 --- a/python/sglang/srt/models/gigachat35.py +++ b/python/sglang/srt/models/gigachat35.py @@ -584,7 +584,12 @@ class GigaChat35Model(nn.Module): hidden_states, residual = self.layers[ self.end_layer - 1 - ].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch) + ].layer_communicator.finish_layer_stack( + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, + ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 218447e353..8bb5013648 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -1134,7 +1134,10 @@ class Glm4MoeModel(nn.Module): last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index c25925d36c..107ae88c98 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -833,7 +833,10 @@ class Glm4MoeLiteModel(nn.Module): last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 525809e395..b5d310c773 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -1191,7 +1191,10 @@ class Glm5NextModel(nn.Module): last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: if self.config.mhc: diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index 669ab0566b..27e9a5ccf7 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -732,7 +732,10 @@ class GptOssModel(nn.Module): ) last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 1264eded30..62ba0fc8cb 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -611,7 +611,10 @@ class LagunaModel(nn.Module): last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index 22cbdcb75b..0e503e67e9 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -766,7 +766,12 @@ class LLaDA2MoeModel(nn.Module): hidden_states, residual = self.layers[ self.end_layer - 1 - ].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch) + ].layer_communicator.finish_layer_stack( + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, + ) if not self.pp_group.is_last_rank: return PPProxyTensors( { diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 02af86b11c..353355d40a 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -1081,7 +1081,10 @@ class MiMoV2Model(nn.Module): last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) # A draft targeting the final layer ("after layer diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index e1150472e2..67fe8d0aa5 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -1184,7 +1184,10 @@ class MiniMaxM2Model(nn.Module): last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index b8cab5d3c4..08c01dd606 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -1522,7 +1522,10 @@ class MiniMaxM3Model(nn.Module): last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index a63c59a472..9540d79ce3 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -760,7 +760,10 @@ class NemotronHModel(nn.Module): last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index a8f2ecc5ba..afe1a56cac 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -1220,7 +1220,10 @@ class Qwen2MoeModel(nn.Module): last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 1658edd2b5..0fb9626600 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -1746,8 +1746,6 @@ class Qwen3_5ForCausalLM(nn.Module): rms_epsilon=config.rms_norm_eps, # Every layer was checked above. can_defer_finalize=lambda layer: True, - # The final GemmaRMSNorm closes out the last layer's handoff. - final_norm_consumes_handoff=True, label="Qwen3.5", ) @@ -1870,6 +1868,7 @@ class Qwen3_5ForCausalLM(nn.Module): # The final norm below finalizes a deferred MoE output in its kernel. final_norm_takes_handoff=self.flashinfer_mnnvl_cutedsl_fusion is not None and self.pp_group.is_last_rank, + preserve_declared=not self.pp_group.is_last_rank, ) # Return intermediate tensors for pipeline parallelism diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index df6f19ed56..a644956def 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -1258,7 +1258,12 @@ class Qwen3LLMModel(Qwen3Model): hidden_states, residual = self.layers[ self.end_layer - 1 - ].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch) + ].layer_communicator.finish_layer_stack( + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, + ) # Handle deepstack for the last processed layer (HF-order path only). last_deepstack = ( diff --git a/python/sglang/srt/models/qwen3_vl_moe.py b/python/sglang/srt/models/qwen3_vl_moe.py index 22035e14e6..da7a93a4ca 100644 --- a/python/sglang/srt/models/qwen3_vl_moe.py +++ b/python/sglang/srt/models/qwen3_vl_moe.py @@ -158,7 +158,10 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index 6f2e14886d..6de760f7df 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -1167,7 +1167,10 @@ class SarvamMLAModel(nn.Module): last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py index ea7f7f50c8..843046cdaf 100644 --- a/python/sglang/srt/models/sdar.py +++ b/python/sglang/srt/models/sdar.py @@ -421,7 +421,12 @@ class SDARModel(nn.Module): hidden_states, residual = self.layers[ self.end_layer - 1 - ].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch) + ].layer_communicator.finish_layer_stack( + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, + ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index 9479df3d6d..a7cd5b6be8 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -486,7 +486,10 @@ class SDARMoeModel(nn.Module): last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 228d364d0b..4c53cba411 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -716,7 +716,10 @@ class Step3p5Model(nn.Module): # break last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, ) if not self.pp_group.is_last_rank: return PPProxyTensors( diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index f51da451e0..d956299d06 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -1662,7 +1662,12 @@ class XllmModel(nn.Module): hidden_states, residual = self.layers[ self.end_layer - 1 - ].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch) + ].layer_communicator.finish_layer_stack( + hidden_states, + residual, + forward_batch, + preserve_declared=not self.pp_group.is_last_rank, + ) if not self.pp_group.is_last_rank: return PPProxyTensors( { diff --git a/test/registered/ops/test_aiter_allreduce_fusion_amd.py b/test/registered/ops/test_aiter_allreduce_fusion_amd.py index ccddf7aecd..53d4412c4b 100755 --- a/test/registered/ops/test_aiter_allreduce_fusion_amd.py +++ b/test/registered/ops/test_aiter_allreduce_fusion_amd.py @@ -5,13 +5,16 @@ import sys import tempfile import types import unittest -from contextlib import ExitStack from pathlib import Path +from unittest.mock import MagicMock, patch import torch -from sglang.srt.layers.communicator import LayerCommunicator +from sglang.srt.layers.communicator import UnreducedOutput +from sglang.srt.layers.communicator.layer import LayerCommunicator +from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.runtime_context import get_context, get_flags, get_parallel from sglang.test.ci.ci_register import register_amd_ci from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase, publish_build_topology @@ -348,15 +351,6 @@ class TestAiterAllreduceFusionAmd(unittest.TestCase): ) -def _fake_self(*, is_last_layer=False, tp_size=8): - """Minimal stand-in for a LayerCommunicator with the fields the gate reads.""" - return types.SimpleNamespace( - _speculative_algo=None, - is_last_layer=is_last_layer, - _context=types.SimpleNamespace(tp_size=tp_size), - ) - - def _fake_forward_batch(batch_size=8, forward_mode=ForwardMode.DECODE): return types.SimpleNamespace( input_ids=types.SimpleNamespace(shape=(batch_size,)), @@ -365,121 +359,80 @@ def _fake_forward_batch(batch_size=8, forward_mode=ForwardMode.DECODE): class TestAiterAllreduceFusionGate(CustomTestCase): - """Pure-logic coverage of the aiter all-reduce + RMSNorm fusion gate. + """Consumer fusion guards and the producer's expert-parallel gate. - Covers ``LayerCommunicator.should_fuse_mlp_allreduce_with_next_layer``, - specifically the AMD/aiter branch guards that disable the fused path under - DP attention or an expert-parallel A2A backend (e.g. mori). Without those - guards the fused custom all-reduce is invoked during CUDA graph capture in - those configs and crashes in ``custom_all_reduce.flush_graph_buffers``. - - The gate is pure decision logic, so the test stubs out the module-level - dependencies and invokes the method on a minimal fake instance. No GPU or - distributed initialization is required. + DP attention must not invoke the custom all-reduce, while an A2A MoE must + finish in its combine path instead of leaving a TP sum to this consumer. + These checks use CPU tensors and do not initialize distributed groups. """ def _evaluate_gate( self, *, dp_attention, - a2a_is_none, aiter_enabled=True, use_aiter=True, tp_world_size=8, - is_last_layer=False, - tp_size=8, forward_mode=ForwardMode.DECODE, disable_in_prefill=False, disable_in_decode=False, + matching_group=True, ): - """Run the gate with the aiter branch isolated (flashinfer forced off).""" - a2a_backend = types.SimpleNamespace(is_none=lambda: a2a_is_none) - - with ExitStack() as stack: - stack.enter_context( - patch_communicator("is_enable_moe_cp_allgather", lambda: False) - ) - stack.enter_context( - patch_communicator( - "get_attn_tp_context", - lambda: types.SimpleNamespace(input_scattered=False), - ) - ) - # Force the NVIDIA/flashinfer term off so the aiter branch decides. - stack.enter_context( - patch_communicator( - "apply_flashinfer_allreduce_fusion", lambda batch_size: False - ) - ) - stack.enter_context(patch_communicator("_use_aiter", use_aiter)) - # moe_ep_size/moe_tp_size of 1 keep the hybrid EP+TP guard inactive - # so the aiter branch is what decides. - stack.enter_context( - patch_communicator( - "get_parallel", - lambda: types.SimpleNamespace( - tp_size=tp_world_size, moe_ep_size=1, moe_tp_size=1 - ), - ) - ) - # the gate reads get_exec().comm.enable_aiter_allreduce_fusion - from sglang.srt.runtime_context import get_context, get_flags - - stack.enter_context( - get_context().override_server_args( - enable_aiter_allreduce_fusion=aiter_enabled, - disable_aiter_allreduce_fusion_in_prefill=disable_in_prefill, - disable_aiter_allreduce_fusion_in_decode=disable_in_decode, - ) - ) - - stack.enter_context(get_flags().dp.override(enabled=dp_attention)) - stack.enter_context( - patch_communicator("get_moe_a2a_backend", lambda: a2a_backend) - ) - - fake_self = _fake_self(is_last_layer=is_last_layer, tp_size=tp_size) - return LayerCommunicator.should_fuse_mlp_allreduce_with_next_layer( - fake_self, _fake_forward_batch(forward_mode=forward_mode) + group = object() + norm = types.SimpleNamespace( + forward_with_allreduce_fusion=MagicMock(return_value=("norm", "residual")) + ) + plan = types.SimpleNamespace( + input_layernorm=norm, _attn_input_fuses_quant=False + ) + hidden = torch.ones(8, 32) + owed = UnreducedOutput(hidden, group=group if matching_group else object()) + with ( + patch_communicator("apply_flashinfer_allreduce_fusion", return_value=False), + patch_communicator("post_experts_reduction_group", return_value=group), + patch_communicator("_use_aiter", use_aiter), + get_parallel().override(tp_size=tp_world_size), + get_context().override_server_args( + enable_aiter_allreduce_fusion=aiter_enabled, + disable_aiter_allreduce_fusion_in_prefill=disable_in_prefill, + disable_aiter_allreduce_fusion_in_decode=disable_in_decode, + ), + get_flags().dp.override(enabled=dp_attention), + ): + result = LayerCommunicator._reduce_output_and_update_and_read_residual( + plan, + owed, + hidden.clone(), + _fake_forward_batch(forward_mode=forward_mode), + None, ) + self.assertEqual(norm.forward_with_allreduce_fusion.called, result is not None) + return result is not None def test_dense_tp_fuses(self): # Baseline supported path: dense TP, no DP attention, no EP backend. - self.assertTrue(self._evaluate_gate(dp_attention=False, a2a_is_none=True)) + self.assertTrue(self._evaluate_gate(dp_attention=False)) def test_dp_attention_disables_fusion(self): # The fix: DP attention has no dense TP all-reduce to fuse. - self.assertFalse(self._evaluate_gate(dp_attention=True, a2a_is_none=True)) + self.assertFalse(self._evaluate_gate(dp_attention=True)) - def test_ep_backend_disables_fusion(self): - # The fix: with an EP A2A backend (e.g. mori) the reduction lives in - # combine(), not a TP all-reduce. - self.assertFalse(self._evaluate_gate(dp_attention=False, a2a_is_none=False)) + def test_ep_backend_does_not_leave_a_tp_sum(self): + # The producer rejects A2A before the consumer can select fusion. + # In particular, Mori combine owns the output's reduction. + backend = types.SimpleNamespace(is_none=lambda: False) + with patch.object(moe_utils, "get_moe_a2a_backend", return_value=backend): + self.assertFalse(moe_utils.post_experts_sum_is_one_all_reduce()) - def test_dp_attention_and_ep_disables_fusion(self): - # The crashing config from the TP8+EP8+mori repro. - self.assertFalse(self._evaluate_gate(dp_attention=False, a2a_is_none=False)) - self.assertFalse(self._evaluate_gate(dp_attention=True, a2a_is_none=False)) + def test_another_reduction_group_does_not_fuse(self): + self.assertFalse(self._evaluate_gate(dp_attention=False, matching_group=False)) def test_flag_off_disables_fusion(self): # Sanity: the gate still respects the opt-in flag on the dense path. - self.assertFalse( - self._evaluate_gate( - dp_attention=False, a2a_is_none=True, aiter_enabled=False - ) - ) + self.assertFalse(self._evaluate_gate(dp_attention=False, aiter_enabled=False)) - def test_last_layer_disables_fusion(self): - self.assertFalse( - self._evaluate_gate( - dp_attention=False, a2a_is_none=True, is_last_layer=True - ) - ) - - def test_tp1_disables_fusion(self): - self.assertFalse( - self._evaluate_gate(dp_attention=False, a2a_is_none=True, tp_size=1) - ) + def test_tp6_does_not_use_aiter_fusion(self): + self.assertFalse(self._evaluate_gate(dp_attention=False, tp_world_size=6)) PREFILL_MODES = (ForwardMode.EXTEND, ForwardMode.MIXED, ForwardMode.SPLIT_PREFILL) DECODE_MODES = ( @@ -495,7 +448,6 @@ class TestAiterAllreduceFusionGate(CustomTestCase): self.assertFalse( self._evaluate_gate( dp_attention=False, - a2a_is_none=True, forward_mode=mode, disable_in_prefill=True, ) @@ -505,7 +457,6 @@ class TestAiterAllreduceFusionGate(CustomTestCase): self.assertTrue( self._evaluate_gate( dp_attention=False, - a2a_is_none=True, forward_mode=mode, disable_in_prefill=True, ) @@ -517,7 +468,6 @@ class TestAiterAllreduceFusionGate(CustomTestCase): self.assertFalse( self._evaluate_gate( dp_attention=False, - a2a_is_none=True, forward_mode=mode, disable_in_decode=True, ) @@ -527,7 +477,6 @@ class TestAiterAllreduceFusionGate(CustomTestCase): self.assertTrue( self._evaluate_gate( dp_attention=False, - a2a_is_none=True, forward_mode=mode, disable_in_decode=True, ) @@ -539,7 +488,6 @@ class TestAiterAllreduceFusionGate(CustomTestCase): self.assertTrue( self._evaluate_gate( dp_attention=False, - a2a_is_none=True, forward_mode=mode, ) ) diff --git a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py index c6d15f9605..94c7a160de 100644 --- a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py @@ -1,3 +1,4 @@ +import unittest from functools import partial from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -8,7 +9,6 @@ import torch from sglang.srt.layers.communicator import ( ADD, NORM_QUANT_READ, - FfnExitFusion, LayerCommunicator, Layout, SumGroup, @@ -56,7 +56,6 @@ def _communicator(): comm.post_attention_layernorm = RMSNorm(8, eps=1e-6) comm.enable_fused_ar_quant = False comm._attn_input_fusions = comm._select_attn_input_fusions() - comm._ffn_exit_fusions = comm._select_ffn_exit_fusions() # Only the ordinary batches' attention input half, with these entries. comm._context = SimpleNamespace() comm._sp_steps = comm._input_scattered_steps = comm._cp_steps = None @@ -75,6 +74,7 @@ def _communicator(): carried_fusions=comm._attn_input_fusions, ), input_move=None, + input_sum=None, handoff=None, ) ) @@ -127,7 +127,6 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): ) ), hands_off_finalize=False, - next_input_absorbs=False, output_is_replicated=False, ) hidden_states = UnreducedOutput(torch.zeros(8, 8)) @@ -143,8 +142,7 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): ) assert torch.equal(out_hidden, torch.ones(8, 8)) - defer, absorb, _ = last._ffn_exit_fusions - assert defer(_DECODE) is None and absorb(_DECODE) is None + assert not last.fusions.can_defer_finalize(last, _DECODE) def test_cutedsl_entries_come_before_the_base_fused_kernel(): @@ -212,54 +210,51 @@ def test_the_fusion_runs_only_on_the_ffn_full_rows(): assert fusion._common_eligible(comm, _DECODE, 8) is eligible -def test_the_exit_tries_the_handoff_then_the_absorb_then_the_base_kernel(eligible): +def test_finalize_handoff_is_a_producer_capability(eligible): comm = _communicator() fusion = comm.fusions - defer, absorb, base = comm._ffn_exit_fusions - assert _bound(defer) == (fusion._defer_moe_finalize, (comm,)) - assert _bound(absorb) == (fusion._absorb_all_reduce, (comm,)) - assert base == comm._next_input_norm_takes_ffn_sum - # Before install() chooses them, neither CuTe DSL kernel takes the sum. - assert defer(_DECODE) is None and absorb(_DECODE) is None + assert not fusion.can_defer_finalize(comm, _DECODE) fusion.install( - SimpleNamespace(), - hands_off_finalize=True, - next_input_absorbs=True, - output_is_replicated=False, + SimpleNamespace(), hands_off_finalize=True, output_is_replicated=False ) with patch.object(CuteDSLFusion, "_should_use_finalize", return_value=True): - assert defer(_DECODE) is FfnExitFusion.DEFER_MOE_FINALIZE - assert absorb(_DECODE) is FfnExitFusion.NEXT_INPUT + assert fusion.can_defer_finalize(comm, _DECODE) + fusion.requires_local_reduction = True + assert not fusion.can_defer_finalize(comm, _DECODE) + + +def test_install_does_not_require_a_fused_successor(): + reset_context() + publish(ServerArgs(model_path="dummy"), role="test") + first, last = _communicator(), _communicator() + ordinary = SimpleNamespace(layer_communicator=SimpleNamespace(fusions=None)) + install_cutedsl_fusion( + [ + SimpleNamespace(layer_communicator=first), + ordinary, + SimpleNamespace(layer_communicator=last), + ], + hidden_size=8, + top_k=2, + rms_epsilon=1e-6, + can_defer_finalize=lambda layer: True, + label="test", + ) + assert first.fusions.hands_off_finalize + assert last.fusions.hands_off_finalize def test_a_replicated_output_producer_keeps_its_own_all_reduce(eligible): - """A TP1 shared expert (or TP1 dense MLP) is not partial; handing its layer's - reduction onward would scale the replicated output by tp_size.""" layers = [ - SimpleNamespace(layer_communicator=_communicator(), replicated=replicated) - for replicated in (True, False, False) + SimpleNamespace(layer_communicator=_communicator(), replicated=value) + for value in (True, False) ] _install(layers, requires_local_reduction=lambda layer: layer.replicated) - replicated, plain = (layer.layer_communicator for layer in layers[:2]) - - def exit_fusion(communicator): - return next( - filter(None, (fused(_DECODE) for fused in communicator._ffn_exit_fusions)), - None, - ) - - with patch.object( - LayerCommunicator, - "should_fuse_mlp_allreduce_with_next_layer", - return_value=False, - ): - assert exit_fusion(replicated) is None - assert exit_fusion(plain) is FfnExitFusion.NEXT_INPUT - # Consuming what a predecessor skipped stays independently eligible. - assert ( - replicated.fusions._can_consume_post_moe_all_reduce(replicated, _DECODE, 8) - is True - ) + replicated, plain = [layer.layer_communicator for layer in layers] + assert replicated.fusions.requires_local_reduction + assert not plain.fusions.requires_local_reduction + # The restriction belongs to the producer; incoming sums can still fuse. + assert replicated.fusions._can_consume_post_moe_all_reduce(replicated, _DECODE, 8) def test_a_service_nested_under_a_wrapper_is_prepared(): @@ -548,6 +543,60 @@ def test_retargeted_profiles_cover_capacity_and_pass_flashinfer_validation( ) +class TestDeferredLoraAllReduce(unittest.TestCase): + def test_installed_cutedsl_provider_restores_deferred_sum(self): + from types import SimpleNamespace + from unittest.mock import patch + + from sglang.srt.layers.communicator import layer as exits + from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion + + group = object() + fb = SimpleNamespace( + input_ids=torch.zeros(2), residual_stream=SimpleNamespace(residual=None) + ) + provider = CuteDSLFusion() + boundary = SimpleNamespace(fusions=provider) + for eligible, a2a in ((True, False), (False, False), (True, True)): + with ( + patch.object(exits, "_ffn_has_tokens", return_value=True), + patch.object( + exits, "post_experts_sum_is_one_all_reduce", return_value=False + ), + patch.object( + exits, "get_lora", return_value=SimpleNamespace(enable_lora=True) + ), + patch.object( + exits, + "get_exec", + return_value=SimpleNamespace( + comm=SimpleNamespace(enable_quant_communications=False) + ), + ), + patch.object( + exits, + "get_moe_a2a_backend", + return_value=SimpleNamespace(is_none=lambda: not a2a), + ), + patch.object( + exits, "get_parallel", return_value=SimpleNamespace(tp_group=group) + ), + patch.object(exits, "post_experts_reduction_group", return_value=group), + patch.object( + exits, "apply_flashinfer_allreduce_fusion", return_value=False + ), + patch.object( + provider, "can_defer_all_reduce", return_value=eligible + ) as gate, + ): + # Avoid platform-specific Aiter details: no residual cannot fuse. + with patch.object(exits, "_use_aiter", False, create=True): + result = exits._can_defer_ffn_reduction(fb, boundary) + self.assertEqual(result, eligible and not a2a) + if a2a: + gate.assert_not_called() + + if __name__ == "__main__": import sys diff --git a/test/registered/unit/layers/test_batch_owned_residual.py b/test/registered/unit/layers/test_batch_owned_residual.py index 2b9e3682ee..b764298456 100644 --- a/test/registered/unit/layers/test_batch_owned_residual.py +++ b/test/registered/unit/layers/test_batch_owned_residual.py @@ -73,6 +73,9 @@ class TestBatchOwnedResidual(CustomTestCase): def test_pp_receive_attaches_and_export_releases_the_same_stream(self): communicator = LayerCommunicator.__new__(LayerCommunicator) + communicator._batch_steps = lambda _: SimpleNamespace( + attention=SimpleNamespace(input_sum=None), ffn=None + ) fb = SimpleNamespace(residual_stream=None) hidden = torch.full((2, 4), 2.0) residual = torch.ones(2, 4) diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 23fbec4b6a..f00f7e92cb 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -115,7 +115,6 @@ def make_communicator( # A subclass decides through its own fused kernels; no TP left to defer # over otherwise. communicator._context = types.SimpleNamespace(tp_size=1) - communicator._ffn_exit_fusions = communicator._select_ffn_exit_fusions() communicator._ffn_leaves_sum_to_reduce_scatter = MagicMock( return_value=reduce_scatter ) @@ -247,48 +246,23 @@ class TestFfnExit(CustomTestCase): def test_flags_are_restored_after_the_ffn(self): before = published_flags() - self.run_exit(make_communicator(fuse=True, reduce_scatter=True)) - self.assertEqual(published_flags(), before) + for fuse, reduce_scatter in ((True, False), (False, True)): + self.run_exit(make_communicator(fuse=fuse, reduce_scatter=reduce_scatter)) + self.assertEqual(published_flags(), before) - def test_the_next_input_s_fused_kernel_is_tried_first(self): - """The base exit asks the next layer's AR + add + norm before the - unfused condition, and leaves the sum to it when it takes the batch.""" - for takes in (False, True): - with self.subTest(takes=takes): - communicator = make_communicator(fuse=False, reduce_scatter=False) - communicator.should_fuse_mlp_allreduce_with_next_layer = MagicMock( - return_value=takes - ) - communicator._ffn_exit_fusions = communicator._select_ffn_exit_fusions() - seen, _ = self.run_exit(communicator) - self.assertEqual(seen, (takes, False)) - communicator.should_fuse_mlp_allreduce_with_next_layer.assert_called_once() - self.assertEqual( - communicator._ffn_sum_moves_to_next_layer.called, not takes - ) - - def test_subclass_decisions_are_used(self): - class NeverDefers(LayerCommunicator): - def should_fuse_mlp_allreduce_with_next_layer(self, forward_batch): - return False - - communicator = make_communicator( - fuse=True, reduce_scatter=False, cls=NeverDefers - ) - seen, _ = self.run_exit(communicator) - self.assertEqual(seen, (False, False)) - communicator._complete_ffn_output_now.assert_called_once() + def test_deferral_does_not_inspect_consumer_fusion(self): + for allows in (False, True): + communicator = make_communicator(fuse=allows, reduce_scatter=False) + seen, _ = self.run_exit(communicator) + self.assertEqual(seen, (allows, False)) + communicator._ffn_sum_moves_to_next_layer.assert_called_once() def test_deferral_implies_fusion_and_passes_the_handoff_through(self): """A deferring communicator publishes both flags, and a non-tensor handoff leaves finish() untouched for the next layer's input norm.""" class Defers(LayerCommunicator): - def _select_ffn_exit_fusions(self): - return (lambda forward_batch: comm.FfnExitFusion.DEFER_MOE_FINALIZE,) - - def should_fuse_mlp_allreduce_with_next_layer(self, forward_batch): - return False + fusions = types.SimpleNamespace(can_defer_finalize=lambda layer, fb: True) communicator = make_communicator(fuse=False, reduce_scatter=False, cls=Defers) handoff = object() diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index e7b5ba5e2a..4a6a95d704 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -1427,7 +1427,9 @@ class TestInputScatteredAttention(CustomTestCase): batch = SimpleNamespace(residual_stream=None) stream = residual_batch.start(batch) stream.write(torch.full((4, HIDDEN), 3.0)) - contribution = stream.leave(torch.ones(4, HIDDEN), ADD) + contribution = stream.leave( + torch.ones(4, HIDDEN), ADD, declared_sum=SumGroup.TP + ) hidden, residual = communicator.prepare_attn(contribution, stream, batch) self.assertEqual(scattered, [4]) # Norm: (2 * (h + r), h + r) on the slice, with h the completed sum. diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py index 24dff2b096..215ac56a8b 100644 --- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py +++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py @@ -331,7 +331,7 @@ class TestFuseMlpAllReduceGate(CustomTestCase): tp_size=moe_ep_size * moe_tp_size * moe_dp_size, ), ): - return LayerCommunicator.should_fuse_mlp_allreduce_with_next_layer( + return LayerCommunicator._ffn_sum_can_move_to_next_layer( _fake_communicator(ffn_sum_is_movable), forward_batch ) @@ -401,6 +401,9 @@ class TestDeferFfnReduction(CustomTestCase): forward_batch = types.SimpleNamespace( input_ids=types.SimpleNamespace(shape=(batch_size,)), global_dp_buffer_len=global_tokens, + residual_stream=types.SimpleNamespace( + residual=torch.ones(max(batch_size, 1), 4) + ), ) with ( patch_communicator( @@ -410,6 +413,21 @@ class TestDeferFfnReduction(CustomTestCase): patch_communicator("is_enable_moe_cp_allgather", return_value=False), patch_communicator("apply_flashinfer_allreduce_fusion", return_value=fused), patch_communicator("_use_aiter", False), + patch_communicator( + "get_lora", return_value=types.SimpleNamespace(enable_lora=lora) + ), + patch_communicator( + "get_exec", + return_value=types.SimpleNamespace( + comm=types.SimpleNamespace( + enable_quant_communications=quant_communications + ) + ), + ), + patch_communicator( + "post_experts_reduction_group", + return_value=tp_group_object if tp_group else object(), + ), patch_communicator( "get_attn_tp_context", return_value=types.SimpleNamespace(input_scattered=False), @@ -450,7 +468,11 @@ class TestDeferFfnReduction(CustomTestCase): return_value=types.SimpleNamespace(enable_lora=lora), ), get_parallel().override( - moe_ep_size=1, moe_tp_size=4, moe_dp_size=1, tp_size=4 + moe_ep_size=1, + moe_tp_size=4, + moe_dp_size=1, + tp_size=4, + tp_group=tp_group_object, ), patch.object( moe_utils, diff --git a/test/registered/unit/layers/test_residual_stream.py b/test/registered/unit/layers/test_residual_stream.py index 0da8eb5905..44eb241df6 100644 --- a/test/registered/unit/layers/test_residual_stream.py +++ b/test/registered/unit/layers/test_residual_stream.py @@ -1,6 +1,6 @@ import unittest from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import Mock, patch import torch @@ -13,13 +13,18 @@ from sglang.srt.layers.communicator import ( StageEntry, StageInput, StageOutput, + SumGroup, make_boundary, ) from sglang.srt.layers.communicator.layer import StageCommunicator from sglang.srt.layers.communicator.output import UnreducedOutput from sglang.srt.layers.communicator.residual.access import add_to_output from sglang.srt.layers.communicator.residual.add_norm import ADD -from sglang.srt.layers.communicator.residual.stream import OwedOutput, ResidualStream +from sglang.srt.layers.communicator.residual.stream import ( + DeclaredSum, + OwedOutput, + ResidualStream, +) from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -138,6 +143,77 @@ class TestResidualStream(CustomTestCase): update=SimpleNamespace(adds_plainly=False), ) + def test_declared_sum_snapshot_and_pipeline_handoff_preserve_the_partial(self): + stream = ResidualStream(self.residual) + hidden = stream.leave(self.partial, ADD, declared_sum=SumGroup.TP) + self.assertIsInstance(hidden, OwedOutput) + with patch( + "sglang.srt.layers.communicator.residual.stream._sum_group", + return_value=self.group, + ): + snapshot = stream.snapshot(hidden) + torch.testing.assert_close(snapshot, torch.full((2, 4), 5.0)) + torch.testing.assert_close(self.partial, torch.ones(2, 4)) + wire, residual = stream.finish(hidden, preserve_declared=True) + self.assertIs(wire, self.partial) + self.assertIsInstance(stream.pending.owed, DeclaredSum) + received, rebuilt = ResidualStream.arrive( + wire, residual, ADD, declared_sum=SumGroup.TP + ) + self.assertIsInstance(received, OwedOutput) + self.assertIs(rebuilt.pending.owed.group, SumGroup.TP) + + def test_materialized_declared_sum_is_not_reduced_again(self): + rows = Layout(frozenset()) + boundary = make_boundary( + EdgeDecl( + StageOutput(rows, group=SumGroup.TP, always_leaves=True), + StageInput(rows), + rows, + rows, + ) + ) + stream = ResidualStream(self.residual) + hidden = stream.leave(self.partial, ADD, declared_sum=SumGroup.TP) + with patch( + "sglang.srt.layers.communicator.residual.stream._sum_group", + return_value=self.group, + ): + hidden = stream.complete(hidden) + + def norm(value, residual): + return value + residual, value + residual + + result, _ = boundary.prepare( + hidden, + self.residual, + None, + norm, + None, + pending=stream.pending, + update=stream.pending.update, + ) + torch.testing.assert_close(result, torch.full((2, 4), 5.0)) + self.group.all_reduce.assert_called_once() + + def test_declared_sum_must_match_the_consumer_contract(self): + rows = Layout(frozenset()) + boundary = make_boundary( + EdgeDecl(StageOutput(rows), StageInput(rows), rows, rows) + ) + stream = ResidualStream(self.residual) + stream.leave(self.partial, ADD, declared_sum=SumGroup.TP) + with self.assertRaisesRegex(RuntimeError, "sum does not match"): + boundary.prepare( + self.partial, + self.residual, + None, + None, + None, + pending=stream.pending, + update=ADD, + ) + def test_snapshot_finishes_a_copy_without_consuming_main_work(self): snapshot = self.stream.snapshot(self.hidden) torch.testing.assert_close(snapshot, torch.full((2, 4), 5.0)) diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py index 4475883ab9..e887e0ed73 100644 --- a/test/registered/unit/lora/test_triton_dp_attention_unit.py +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -244,6 +244,7 @@ def test_communicator_publishes_layout_at_each_transition( ) selected = SimpleNamespace( attention=SimpleNamespace( + input_sum=None, prepare=partial( _consumer_step, step=partial( @@ -260,6 +261,7 @@ def test_communicator_publishes_layout_at_each_transition( handoff=lambda hidden_states, *args: hidden_states, ), ffn=SimpleNamespace( + input_sum=None, prepare=lambda hidden_states, residual, *args, **kwargs: ( hidden_states, residual, diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index 9829acf395..056afaace1 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -83,9 +83,6 @@ class DeferringLayer(nn.Module): ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial, ffn_sum_is_movable=True, ) - self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer = ( - lambda batch: defer - ) self.layer_communicator._ffn_sum_moves_to_next_layer = lambda batch, **_: defer self.layer_communicator.is_last_layer = False self.layer_communicator._sp_steps = None @@ -289,6 +286,9 @@ class TestPipelineResidualReception(CustomTestCase): def test_written_streams_do_not_read_a_separate_residual(self): comm_instance = comm.LayerCommunicator.__new__(comm.LayerCommunicator) + comm_instance._batch_steps = lambda fb: SimpleNamespace( + attention=SimpleNamespace(input_sum=None), ffn=None + ) comm_instance._residual = SimpleNamespace( ffn_update=SimpleNamespace(at_producer=True) ) @@ -303,6 +303,9 @@ class TestPipelineResidualReception(CustomTestCase): def test_optional_residual_and_declared_partial_keep_the_wire_values(self): comm_instance = comm.LayerCommunicator.__new__(comm.LayerCommunicator) + comm_instance._batch_steps = lambda fb: SimpleNamespace( + attention=SimpleNamespace(input_sum=None), ffn=None + ) partial = torch.randn(2, 4) prior = torch.randn_like(partial) hidden, residual = comm_instance.from_pp( diff --git a/test/registered/unit/models/test_layer_stack_exit.py b/test/registered/unit/models/test_layer_stack_exit.py index d2babac9df..cd3258cd9d 100644 --- a/test/registered/unit/models/test_layer_stack_exit.py +++ b/test/registered/unit/models/test_layer_stack_exit.py @@ -76,10 +76,7 @@ def defers(node): through ffn_exit), or wraps its output as unreduced itself.""" if declares_no_deferral(node): return False - return any( - any(calls(node, name)) - for name in ("ffn_exit", "should_fuse_mlp_allreduce_with_next_layer", UNREDUCED) - ) + return any(any(calls(node, name)) for name in ("ffn_exit", UNREDUCED)) class Census: diff --git a/test/registered/unit/models/test_nemotron_h_mtp_reduction.py b/test/registered/unit/models/test_nemotron_h_mtp_reduction.py index fbdea260db..7cdb7b6f0e 100644 --- a/test/registered/unit/models/test_nemotron_h_mtp_reduction.py +++ b/test/registered/unit/models/test_nemotron_h_mtp_reduction.py @@ -6,6 +6,7 @@ import torch from torch import nn from sglang.srt.layers.communicator import ADD +from sglang.srt.layers.communicator.layout import SumGroup from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.models import nemotron_h_mtp @@ -84,9 +85,11 @@ class TestNemotronMTPReduction(CustomTestCase): ) stream = residual_batch.start(batch) stream.write(residual) - partial = stream.leave(partial, ADD) + partial = stream.leave( + partial, ADD, declared_sum=SumGroup.ATTN_TP if tp > 1 else None + ) hidden, output_residual = layer( - inputs_embeds=torch.zeros_like(partial), + inputs_embeds=torch.zeros_like(residual), hidden_states=partial, residual=stream, forward_batch=batch, diff --git a/test/registered/unit/models/test_nemotron_h_stages.py b/test/registered/unit/models/test_nemotron_h_stages.py index dfa084823f..62c997ecf3 100644 --- a/test/registered/unit/models/test_nemotron_h_stages.py +++ b/test/registered/unit/models/test_nemotron_h_stages.py @@ -150,11 +150,11 @@ class TestMixerExit(CustomTestCase): def test_decision_table(self): tp_group = object() attention = Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes(tp=2)) - for always, may, fuses in itertools.product((False, True), repeat=3): + for always, may, movable in itertools.product((False, True), repeat=3): if always and may: continue with self.subTest( - always_leaves=always, leaves_for_next_layer=may, fuses=fuses + always_leaves=always, leaves_for_next_layer=may, movable=movable ): produced = StageOutput( attention, @@ -164,9 +164,8 @@ class TestMixerExit(CustomTestCase): ) communicator = SimpleNamespace( _batch_steps=lambda batch: SimpleNamespace(ffn_output=produced), - should_fuse_mlp_allreduce_with_next_layer=MagicMock( - return_value=fuses - ), + _context=SimpleNamespace(tp_size=2), + _ffn_sum_can_move_to_next_layer=MagicMock(return_value=movable), ) hidden = torch.ones(2, 4) with get_parallel().override(tp_group=tp_group): @@ -174,7 +173,7 @@ class TestMixerExit(CustomTestCase): skipped = should_skip_mlp_all_reduce() output = mixer_exit.finish(hidden) self.assertFalse(should_skip_mlp_all_reduce()) - hands_on = may and fuses + hands_on = may and movable self.assertEqual(mixer_exit.skips_reduction, always or hands_on) self.assertEqual(skipped, always or hands_on) if hands_on: diff --git a/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py index cc0c75284e..ed0f3eef67 100644 --- a/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py +++ b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py @@ -38,7 +38,7 @@ def dense_layer(*, reduce_scatter): ffn_output=SimpleNamespace(update=comm.ADD) ), _declared_ffn_sum=lambda steps, skipped: None, - _leave_ffn_output=lambda h, r, stream, update, declared_sum=None: (h, r), + _leave_ffn_output=lambda h, r, stream, update, declared_sum: (h, r), # A dense layer never leaves its sum to the next layer. _select_ffn_completion=lambda fb: comm.FfnCompletion( defer_moe_finalize=False,