diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index 1dc5e89379..3760041507 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -2862,7 +2862,7 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec ### FFN boundary reduction -Use `--boundary-reduction` to choose which optional FFN output reductions may combine the sum with token redistribution. The selected policy applies to both dense and MoE FFN layers. It currently controls models using independently constructed stage boundaries; models still using `LayerCommunicator` retain their existing reduction policy. +Use `--boundary-reduction` to choose which optional FFN output reductions may combine the sum with token redistribution. The selected policy applies to both dense and MoE FFN layers. | Value | Behavior | | --- | --- | @@ -2874,8 +2874,6 @@ Use `--boundary-reduction` to choose which optional FFN output reductions may co `auto` selects `ar` for dense Qwen3 language backbones (including Qwen3-VL wrappers) and Moss-VL; `rsv` for BailingMoELinear/BailingMoeV2_5, LongCat Flash and Step3-VL; and `rs+rsv` otherwise. Qwen3-MoE, Qwen3-Next and Qwen3.5 are not matched by the dense Qwen3 rule and resolve to `rs+rsv`. -LongCat Flash and Moss-VL resolve these defaults, but still use the layer communicator at this revision and do not yet consume this option. - With `auto`, an independent speculative draft resolves its own model default; same-model MTP inherits the target policy. An explicit value applies to both target and draft. The result is recorded in the `speculative_boundary_reduction` server args field (`no_cli=True`); it has no separate CLI flag or independent override during resolution. This option does not disable required attention or MoE collectives, including expert all-to-all, or select the all-reduce fusion backend. For example, `ar` can still use fused all-reduce plus residual normalization; use `--enforce-disable-flashinfer-allreduce-fusion` to disable that FlashInfer fusion. diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py index 9b98e144e2..4253e2969b 100644 --- a/python/sglang/srt/arg_groups/fields/exec_.py +++ b/python/sglang/srt/arg_groups/fields/exec_.py @@ -626,9 +626,8 @@ class ExecComm(msgspec.Struct): help="Select FFN boundary reduction: ar uses all-reduce then token " "redistribution; rs and rsv permit fixed-size and variable-size " "reduce-scatter respectively; rs+rsv permits both (RSv first). " - "Unsupported paths fall back to ar. The option applies only to FFN " - "layers using stage boundaries, where auto resolves the model default. " - "Required attention and MoE collectives and " + "Unsupported paths fall back to ar. auto resolves the model default " + "for all FFN layers. Required attention and MoE collectives and " "all-reduce fusion are unaffected.", resolvable=True, ), diff --git a/python/sglang/srt/arg_groups/layernorm_sp_hook.py b/python/sglang/srt/arg_groups/layernorm_sp_hook.py index d601948261..ddc14ddbaf 100644 --- a/python/sglang/srt/arg_groups/layernorm_sp_hook.py +++ b/python/sglang/srt/arg_groups/layernorm_sp_hook.py @@ -15,7 +15,7 @@ def handle_layernorm_sp(server_args: ServerArgs) -> None: """Validate --enable-layernorm-sp against the resolved parallelism config. Runs in the resolution pipeline rather than in the layers so a model that - never builds a LayerCommunicator rejects the flag instead of ignoring it. + does not use layer boundaries rejects the flag instead of ignoring it. """ cfg = resolving_view(server_args) if not cfg.enable_layernorm_sp: diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index 6404bb7828..3399f0950e 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -18,6 +18,7 @@ from sglang.srt.layers.communicator import ( Layout, tbo_split_moves, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.communicator.residual.access import finish_layer_stack from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.layers.moe import ( @@ -930,6 +931,67 @@ def _compute_extend_num_tokens(input_ids, forward_mode: ForwardMode): # -------------------------------- Execution --------------------------------------- +def model_forward_stages( + layers, + enable_tbo: bool, + positions: torch.Tensor, + forward_batch: ForwardBatch, + hidden_states: torch.Tensor, + zero_allocator: Optional[BumpAllocator] = None, +): + """Run stage operations with an independent residual stream per microbatch.""" + strategy = OperationsStrategy.init_new_tbo( + layers, forward_batch.global_forward_mode + ) + inputs = dict( + positions=positions, + hidden_states=hidden_states, + forward_batch=forward_batch, + zero_allocator=zero_allocator, + ) + if not enable_tbo: + return execute_operations(inputs, strategy.operations)["hidden_states"] + + stream = residual_batch.current(forward_batch) + pending = stream.pending + hidden_states, residual = stream.finish(hidden_states) + inputs["hidden_states"] = hidden_states + parts = _model_forward_tbo_split_inputs( + **inputs, residual=residual, layer_input_rows=layers[0].attn_stage.input_rows + ) + for part in parts: + part["hidden_states"], child_stream = ResidualStream.arrive( + part["hidden_states"], + part.pop("residual"), + pending.update if pending is not None else None, + ) + part["forward_batch"].residual_stream = child_stream + + original_len = hidden_states.shape[0] + forward_batch.residual_stream = None + del inputs, pending, stream, hidden_states, residual + + context = ( + empty_context() + if _is_hip + else deep_gemm_wrapper.configure_deep_gemm_num_sms(strategy.deep_gemm_num_sms) + ) + with context: + outputs = execute_overlapped_operations( + inputs_arr=parts, + operations_arr=[strategy.operations] * 2, + delta_stages=[0, strategy.tbo_delta_stages], + ) + for output in outputs: + output["residual"] = residual_batch.current(output["forward_batch"]) + hidden_states, forward_batch.residual_stream = _model_forward_tbo_merge_outputs( + *outputs, original_len + ) + for output in outputs: + output["forward_batch"].residual_stream = None + return hidden_states + + def model_forward_maybe_tbo( layers, enable_tbo: bool, diff --git a/python/sglang/srt/layers/communicator/__init__.py b/python/sglang/srt/layers/communicator/__init__.py index 57e5e7a0a6..0829ab71a1 100644 --- a/python/sglang/srt/layers/communicator/__init__.py +++ b/python/sglang/srt/layers/communicator/__init__.py @@ -79,8 +79,8 @@ from sglang.srt.layers.communicator.residual.mhc import ( __all__ = [ "LayerCommunicator", - "MHCLayerCommunicator", "LayerFacts", + "MHCLayerCommunicator", "declare_attn", "declare_ffn", "make_attn_stage", diff --git a/python/sglang/srt/layers/communicator/exit.py b/python/sglang/srt/layers/communicator/exit.py index 4bb30f9343..4f8dbe2093 100644 --- a/python/sglang/srt/layers/communicator/exit.py +++ b/python/sglang/srt/layers/communicator/exit.py @@ -191,17 +191,19 @@ class OutputBoundary: dp_step=dp_step, ) steps = self.plan._batch_steps(forward_batch) - if not steps.ffn_output.leaves_for_next_layer: + defer_moe_finalize = ( + self.plan.fusions is not None + and self.plan.fusions.can_defer_finalize(self.plan, forward_batch) + ) + if not steps.ffn_output.leaves_for_next_layer and not ( + self.plan.is_last_layer and defer_moe_finalize + ): return FfnCompletion( defer_moe_finalize=False, fuse_mlp_allreduce=False, mlp_reduce_scatter=mlp_reduce_scatter, complete=complete_now, ) - defer_moe_finalize = ( - self.plan.fusions is not None - and self.plan.fusions.can_defer_finalize(self.plan, forward_batch) - ) # Producers declare remaining work independently of the kernel chosen # by the consumer. Every handoff also carries an unfused completion. fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer( diff --git a/python/sglang/srt/layers/communicator/fusions/cutedsl.py b/python/sglang/srt/layers/communicator/fusions/cutedsl.py index ee6d1992fa..a6ec616b43 100644 --- a/python/sglang/srt/layers/communicator/fusions/cutedsl.py +++ b/python/sglang/srt/layers/communicator/fusions/cutedsl.py @@ -16,7 +16,6 @@ import torch from sglang.srt.layers.communicator import ( FusedMlpInput, HandoffOutput, - LayerCommunicator, SumGroup, TokenAxis, UnreducedOutput, @@ -196,7 +195,7 @@ class CuteDSLFusionService: class CuteDSLFusion: - """One layer's CuTe DSL kernels, given to its LayerCommunicator at + """One layer's CuTe DSL kernels, given to its stage at construction. At the attention input: the MoE finalize + AR + add + norm of a handoff the previous layer left, and the AR + add + norm of a sum it left. At the FFN input: the attention output's AR + add + norm. At the FFN exit: @@ -208,6 +207,7 @@ class CuteDSLFusion: self.service: CuteDSLFusionService | None = None # Producer capabilities; consumer kernel selection is independent. self.hands_off_finalize = False + self.terminal_finalize = False self.requires_local_reduction = False def install( @@ -216,25 +216,27 @@ class CuteDSLFusion: *, hands_off_finalize: bool, output_is_replicated: bool, + terminal_finalize: bool = False, ) -> None: """Install the shared workspace and this producer's capabilities.""" self.service = service self.hands_off_finalize = hands_off_finalize + self.terminal_finalize = terminal_finalize self.requires_local_reduction = output_is_replicated - def attention_input(self, layer: LayerCommunicator) -> tuple: + def attention_input(self, layer) -> tuple: return ( partial(self._finalize_output_and_update_and_read_residual, layer), partial(self._reduce_output_and_update_and_read_residual, layer), ) - def ffn_input(self, layer: LayerCommunicator) -> tuple: + def ffn_input(self, layer) -> tuple: parallel = get_parallel() if ( TokenAxis.ATTN_TP_SCATTER not in layer.input_rows.sharded # The workspace sums over TP, which is then the attention-TP group. and parallel.attn_tp_size == parallel.tp_size - and _fused_norm_gamma(layer.post_attention_layernorm) is not None + and _fused_norm_gamma(layer.norm) is not None ): return ( FusedMlpInput( @@ -255,7 +257,7 @@ class CuteDSLFusion: handoff is completed by its producer's own tail.""" if not isinstance(owed, MoeFinalizeHandoff): return None - gamma = _fused_norm_gamma(layer.input_layernorm) + gamma = _fused_norm_gamma(layer.norm) if gamma is None or not self._should_use_finalize(layer, forward_batch, owed.m): return None if post_residual_addition is not None: @@ -280,7 +282,7 @@ class CuteDSLFusion: return self.service.all_reduce_residual_rms_norm( local_contribution=owed.partial, residual=residual, - gamma=_fused_norm_gamma(layer.input_layernorm), + gamma=_fused_norm_gamma(layer.norm), ) def _mlp_input_reduce_output_and_update_and_read_residual( @@ -298,7 +300,7 @@ class CuteDSLFusion: return self.service.all_reduce_residual_rms_norm( local_contribution=hidden_states, residual=residual, - gamma=_fused_norm_gamma(layer.post_attention_layernorm), + gamma=_fused_norm_gamma(layer.norm), ) def _should_use_finalize(self, layer, forward_batch: ForwardBatch, m: int) -> bool: @@ -313,7 +315,7 @@ class CuteDSLFusion: """Incoming, and independent of this layer's own successor.""" return ( self._common_eligible(layer, forward_batch, m) - and _fused_norm_gamma(layer.input_layernorm) is not None + and _fused_norm_gamma(layer.norm) is not None and not get_exec().comm.enable_quant_communications ) @@ -334,7 +336,11 @@ class CuteDSLFusion: """Whether this producer can emit a handoff with an unfused fallback.""" return ( self.hands_off_finalize - and not self.requires_local_reduction + and (not layer.is_last_layer or self.terminal_finalize) + and ( + not self.requires_local_reduction + or (layer.is_last_layer and self.terminal_finalize) + ) and self._should_use_finalize( layer, forward_batch, int(forward_batch.input_ids.shape[0]) ) @@ -355,12 +361,12 @@ class CuteDSLFusion: and parallel.tp_size > 1 # The FFN runs on the full rows, not each rank's own slice. and TokenAxis.ATTN_TP_SCATTER - not in layer.ffn.entry(forward_batch).input_rows.sharded + not in layer.fusion_rows(forward_batch).sharded ) def _fusion_of(layer: torch.nn.Module) -> CuteDSLFusion | None: - fusions = getattr(layer.layer_communicator, "fusions", None) + fusions = getattr(layer.__dict__.get("ffn_stage"), "fusions", None) return fusions if isinstance(fusions, CuteDSLFusion) else None @@ -373,10 +379,11 @@ def install_cutedsl_fusion( can_defer_finalize: _LayerPredicate, requires_local_reduction: _LayerPredicate | None = None, label: str, + terminal_finalize: bool = False, ) -> CuteDSLFusionService | None: """One shared workspace handle per fusion-enabled layer, or None. - Every entry of ``layers`` must carry a ``layer_communicator``. + Every entry of ``layers`` carries its attention and FFN stages. """ if get_flags().moe.in_speculative_scope: # A draft shares the target's process, which holds one workspace. @@ -389,8 +396,8 @@ def install_cutedsl_fusion( # The workspace compiles one epsilon and cannot see a per-layer one. for layer in fusion_layers: for norm in ( - layer.layer_communicator.input_layernorm, - layer.layer_communicator.post_attention_layernorm, + layer.attn_stage.norm, + layer.ffn_stage.norm, ): if _fused_norm_gamma(norm) is None: continue @@ -416,6 +423,7 @@ def install_cutedsl_fusion( fusion.install( service, hands_off_finalize=hands_off_finalize, + terminal_finalize=terminal_finalize, output_is_replicated=requires_local_reduction is not None and bool(requires_local_reduction(layer)), ) @@ -441,11 +449,9 @@ def prepare_cutedsl_fusion( fusions = [ fusion for module in model.modules() - # Most modules carry no layer_communicator. + # Only decoder modules carry an FFN stage. if isinstance( - fusion := getattr( - module.__dict__.get("layer_communicator"), "fusions", None - ), + fusion := getattr(module.__dict__.get("ffn_stage"), "fusions", None), CuteDSLFusion, ) ] diff --git a/python/sglang/srt/layers/communicator/residual/batch.py b/python/sglang/srt/layers/communicator/residual/batch.py index c6d15293db..70cdb7dd90 100644 --- a/python/sglang/srt/layers/communicator/residual/batch.py +++ b/python/sglang/srt/layers/communicator/residual/batch.py @@ -45,11 +45,37 @@ def take_output(hidden_states, forward_batch): return hidden_states -def norm(hidden_states, forward_batch, layernorm, capture_output=None, **read_kwargs): - hidden_states, residual = current(forward_batch).finish(hidden_states) +def norm( + hidden_states, + forward_batch, + layernorm, + capture_output=None, + *, + handoff_norm=None, + skip_empty=False, + **read_kwargs, +): + hidden_states, residual = current(forward_batch).finish( + hidden_states, takes_handoff=handoff_norm is not None + ) # The terminal consumer now owns the pair. Do not keep layer buffers alive # through logits processing or the next forward on this batch. forward_batch.residual_stream = None + from sglang.srt.layers.communicator.output import HandoffOutput + + if isinstance(hidden_states, HandoffOutput): + if residual is None: + raise RuntimeError("invalid final deferred MoE handoff") + if capture_output is not None: + raise RuntimeError( + "final handoff capture requires an explicit capture adapter" + ) + hidden_states, _ = handoff_norm.finalize( + handoff=hidden_states, residual=residual, gamma=layernorm.gemma_weight + ) + return hidden_states + if skip_empty and hidden_states.shape[0] == 0: + return hidden_states return access.norm_output( hidden_states, residual, layernorm, capture_output, **read_kwargs ) diff --git a/python/sglang/srt/layers/layernorm_sp.py b/python/sglang/srt/layers/layernorm_sp.py index 6f3f146bc1..147d4cf44e 100644 --- a/python/sglang/srt/layers/layernorm_sp.py +++ b/python/sglang/srt/layers/layernorm_sp.py @@ -25,7 +25,7 @@ Everything SP lives here so the feature stays decoupled from model code and from - which models opt in (the Qwen3-dense allowlist) and config validation, - the per-forward ``sp_active`` flag (a ForwardFlags bool) read at depth by the - participant linears and the ``LayerCommunicator``, + participant linears and the layer-boundary batch selector, - the entry-scatter / exit-gather collectives, and - the fused matmul + collective fast-paths for the participant linears. @@ -47,7 +47,7 @@ from sglang.srt.runtime_context import ( from sglang.srt.utils.common import ceil_align # Architectures whose decoder layers route attention/MLP through -# ``LayerCommunicator`` with the standard participant linears, and for which SP +# layer boundaries with the standard participant linears, and for which SP # has been validated. Other models reject --enable-layernorm-sp at construction. # The mechanism is generic; extend the allowlist as families are validated. SP_SUPPORTED_ARCHITECTURES = frozenset({"Qwen3ForCausalLM"}) diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py index 2fc88e189d..833fffdd95 100644 --- a/python/sglang/srt/models/deepseek_nextn.py +++ b/python/sglang/srt/models/deepseek_nextn.py @@ -216,27 +216,21 @@ class DeepseekModelNextN(nn.Module): else: hidden_states = self.eh_proj(eh_input) - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) index_topk_share = IndexTopKShareState.from_mtp_carry(forward_batch) with get_global_expert_distribution_recorder().disable_this_region(): - hidden_states, residual, topk_indices = self.decoder( + (hidden_states, topk_indices) = self.decoder( positions, hidden_states, forward_batch, - residual, zero_allocator, prev_topk_indices=index_topk_share.topk_indices, ) - hidden_states, residual = ( - self.decoder.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch - ) - ) + hidden_states = residual_batch.finish(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): - if residual is not None: - hidden_states, _ = self.shared_head.norm(hidden_states, residual) - else: - hidden_states = self.shared_head.norm(hidden_states) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.shared_head.norm + ) index_topk_share.update(topk_indices) index_topk_share.publish() diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 218c087e8e..0b40c72ec9 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -40,7 +40,7 @@ from sglang.kernels.ops.quantization.fp8_kernel import ( from sglang.srt.batch_overlap.single_batch_overlap import SboFlags, compute_overlap_args from sglang.srt.batch_overlap.two_batch_overlap import ( MaybeTboDeepEPDispatcher, - model_forward_maybe_tbo, + model_forward_stages, ) from sglang.srt.configs.model_config import ( compute_mla_mscale_scaling, @@ -68,14 +68,16 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStatePacker, ) from sglang.srt.layers.communicator import ( - LayerCommunicator, - LayerFacts, + declare_attn, + declare_ffn, enable_moe_dense_fully_dp, get_attn_tp_context, + make_stages, ) from sglang.srt.layers.communicator.adapters.context_parallel import ( maybe_prefetch_next_full_attention_kv, ) +from sglang.srt.layers.communicator.residual import access as residual_access from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx from sglang.srt.layers.dcp.planner import ( @@ -2578,13 +2580,12 @@ class DeepseekV2DecoderLayer(nn.Module): is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False) is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False) - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=1 if is_nextn else config.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) + self._stage_enters_stack = (layer_id) == 0 + self._stage_terminal = (layer_id) == ( + 1 if is_nextn else config.num_hidden_layers + ) - 1 + self._stage_previous_sparse = is_previous_layer_sparse + self._stage_next_sparse = is_next_layer_sparse if self.is_layer_sparse: self.mlp = DeepseekV2MoE( @@ -2618,19 +2619,19 @@ class DeepseekV2DecoderLayer(nn.Module): self._gfx95_quant_format = self._detect_gfx95_quant_format() - self.layer_communicator = self._build_layer_communicator( + self.attn_stage, self.ffn_stage = self._build_stages( input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, qkv_latent_func=self.self_attn.prepare_qkv_latent, ) - def _build_layer_communicator( + def _build_stages( self, *, input_layernorm: nn.Module, post_attention_layernorm: nn.Module, qkv_latent_func: Optional[Callable], - allow_deferred_ffn_reduction: bool = True, + output=None, ): """The communicator for this layer's norms; it chooses its boundary steps from them at construction.""" @@ -2644,15 +2645,29 @@ class DeepseekV2DecoderLayer(nn.Module): from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion fusions = CuteDSLFusion() - return LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=input_layernorm, - post_attention_layernorm=post_attention_layernorm, - allow_reduce_scatter=True, - qkv_latent_func=qkv_latent_func, - allow_deferred_ffn_reduction=allow_deferred_ffn_reduction, - fusions=fusions, + attn_stage, ffn_stage = make_stages( + ( + declare_attn(), + input_layernorm, + {"qkv_latent_func": qkv_latent_func, "fusions": fusions}, + ), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=self._stage_next_sparse, + output=output, + ), + post_attention_layernorm, + {"fusions": fusions}, + ), + previous=declare_ffn( + sparse=self._stage_previous_sparse, next_sparse=self.is_layer_sparse + ) + if not self._stage_enters_stack + else None, + terminal=self._stage_terminal, ) + return attn_stage, ffn_stage def _detect_gfx95_quant_format(self) -> str: if not _is_gfx95_supported: @@ -2698,7 +2713,6 @@ class DeepseekV2DecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], zero_allocator: BumpAllocator, gemm_output_zero_allocator: BumpAllocator = None, llama_4_scaling: Optional[torch.Tensor] = None, @@ -2706,15 +2720,12 @@ class DeepseekV2DecoderLayer(nn.Module): captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, next_full_attention_layer_id: Optional[int] = None, ) -> torch.Tensor: - hidden_states_orig = self.layer_communicator.buffer(hidden_states) - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - captured_last_layer_outputs=captured_last_layer_outputs, - quant_format=self._resolve_gfx95_quant_format(), - ) + hidden_states_orig = residual_access.buffer(hidden_states) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + captured_last_layer_outputs=captured_last_layer_outputs, + quant_format=self._resolve_gfx95_quant_format(), ) with self.self_attn.maybe_use_decode_attn_tp(forward_batch): @@ -2725,7 +2736,7 @@ class DeepseekV2DecoderLayer(nn.Module): zero_allocator=zero_allocator, llama_4_scaling=llama_4_scaling, input_on_attention_tp_slices=( - self.layer_communicator.input_on_attention_tp_slices + self.attn_stage.input_on_attention_tp_slices ), prev_topk_indices=prev_topk_indices, ) @@ -2739,9 +2750,8 @@ class DeepseekV2DecoderLayer(nn.Module): forward_batch, next_full_attention_layer_id ) - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) if isinstance(self.mlp, DeepseekV2MLP): gemm_output_zero_allocator = None @@ -2759,15 +2769,15 @@ class DeepseekV2DecoderLayer(nn.Module): else: _mlp_ctx = nullcontext() - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit, _mlp_ctx: + with self.ffn_stage.exit(forward_batch) as ffn_exit, _mlp_ctx: hidden_states = self.mlp( hidden_states, forward_batch, gemm_output_zero_allocator, ) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) + hidden_states = ffn_exit.finish(hidden_states) - return hidden_states, residual, topk_indices + return (hidden_states, topk_indices) def op_comm_prepare_attn( self, @@ -2775,12 +2785,11 @@ class DeepseekV2DecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], zero_allocator: BumpAllocator, tbo_subbatch_index: Optional[int] = None, ): - state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = ( - self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch) + state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare( + hidden_states, forward_batch ) if get_moe_a2a_backend().is_mori(): state.num_tokens = hidden_states.shape[0] @@ -2794,25 +2803,21 @@ class DeepseekV2DecoderLayer(nn.Module): ) def op_comm_prepare_mlp(self, state): - state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = ( - self.layer_communicator.prepare_mlp( - state.pop("hidden_states_after_attn"), - state.pop("residual_after_input_ln"), - state.forward_batch, - ) + hidden_states = self.attn_stage.finish( + state.pop("hidden_states_after_attn"), state.forward_batch + ) + state.hidden_states_mlp_input = self.ffn_stage.prepare( + hidden_states, state.forward_batch ) def op_comm_postprocess_layer(self, state): - hidden_states, residual = self.layer_communicator.postprocess_layer( - state.pop("hidden_states_mlp_output"), - state.pop("residual_after_comm_pre_mlp"), - state.forward_batch, + hidden_states = self.ffn_stage.postprocess( + state.pop("hidden_states_mlp_output"), state.forward_batch ) output = dict( positions=state.positions, hidden_states=hidden_states, - residual=residual, forward_batch=state.forward_batch, zero_allocator=state.zero_allocator, tbo_subbatch_index=state.tbo_subbatch_index, @@ -3027,13 +3032,13 @@ class DeepseekV2Model(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) initial_topk_indices = None else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) initial_topk_indices = pp_proxy_tensors.tensors.get("topk_indices") index_topk_share = IndexTopKShareState(forward_batch, initial_topk_indices) if not self.pp_group.is_first_rank: @@ -3103,18 +3108,17 @@ class DeepseekV2Model(nn.Module): ) with ctx: layer = self.layers[i] - hidden_states, residual, topk_indices = layer( + (hidden_states, topk_indices) = layer( positions, hidden_states, forward_batch, - residual, zero_allocator, gemm_output_zero_allocator, llama_4_scaling, prev_topk_indices=index_topk_share.topk_indices, - captured_last_layer_outputs=( - aux_hidden_states if i in self.layers_to_capture else None - ), + captured_last_layer_outputs=aux_hidden_states + if i in self.layers_to_capture + else None, next_full_attention_layer_id=self.next_full_attention_layer_id.get( i ), @@ -3122,28 +3126,17 @@ class DeepseekV2Model(nn.Module): index_topk_share.update(topk_indices) if normal_end_layer != self.end_layer: - hidden_states, residual = model_forward_maybe_tbo( + hidden_states = model_forward_stages( layers=self.layers[normal_end_layer : self.end_layer], enable_tbo=True, positions=positions, forward_batch=forward_batch, hidden_states=hidden_states, - residual=residual, zero_allocator=zero_allocator, ) - last_layer = self.layers[self.end_layer - 1] - hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, - residual, - forward_batch, - preserve_declared=not self.pp_group.is_last_rank, - ) if not self.pp_group.is_last_rank: - proxy_tensors = { - "hidden_states": hidden_states, - "residual": residual, - } + proxy_tensors = residual_batch.to_pp(hidden_states, forward_batch) if ( self.use_dsa and dsa_forward_uses_topk @@ -3164,14 +3157,14 @@ class DeepseekV2Model(nn.Module): topk_indices = hidden_states.new_empty( (0, get_dsa_index_topk(self.config)), dtype=torch.int32 ) - proxy_tensors["topk_indices"] = topk_indices - return PPProxyTensors(proxy_tensors) + proxy_tensors.tensors["topk_indices"] = topk_indices + return proxy_tensors else: + hidden_states = residual_batch.finish(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.norm + ) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index f35a7019dc..4208e5de02 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -42,7 +42,7 @@ from transformers import PretrainedConfig from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz from sglang.srt.batch_overlap.two_batch_overlap import ( MaybeTboDeepEPDispatcher, - model_forward_maybe_tbo, + model_forward_stages, ) from sglang.srt.configs.dots3 import Dots3Config from sglang.srt.distributed import ( @@ -58,9 +58,10 @@ from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer from sglang.srt.layers.communicator import ( - LayerCommunicator, - LayerFacts, + declare_attn, + declare_ffn, enable_moe_dense_fully_dp, + make_stages, ) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled @@ -1542,14 +1543,6 @@ class Dots3DecoderLayer(nn.Module): is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False) is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False) - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=1 if is_nextn else config.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - if self.is_layer_sparse: self.mlp = Dots3MoE( config=config, @@ -1579,11 +1572,21 @@ class Dots3DecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1, ) def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool: @@ -1601,13 +1604,10 @@ class Dots3DecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], zero_allocator: BumpAllocator, ) -> torch.Tensor: - hidden_states, residual = self.layer_communicator.prepare_attn( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) hidden_states = self.self_attn( positions=positions, @@ -1616,18 +1616,17 @@ class Dots3DecoderLayer(nn.Module): zero_allocator=zero_allocator, ) - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + with self.ffn_stage.exit(forward_batch) as ffn_exit: hidden_states = self.mlp( hidden_states, forward_batch, ffn_exit.fuse_mlp_allreduce, ffn_exit.mlp_reduce_scatter, ) - return ffn_exit.finish(hidden_states, residual) + return ffn_exit.finish(hidden_states) def op_comm_prepare_attn( self, @@ -1635,12 +1634,11 @@ class Dots3DecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], zero_allocator: BumpAllocator, tbo_subbatch_index: Optional[int] = None, ): - state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = ( - self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch) + state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare( + hidden_states, forward_batch ) state.update( dict( @@ -1652,12 +1650,11 @@ class Dots3DecoderLayer(nn.Module): ) def op_comm_prepare_mlp(self, state): - state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = ( - self.layer_communicator.prepare_mlp( - state.pop("hidden_states_after_attn"), - state.pop("residual_after_input_ln"), - state.forward_batch, - ) + hidden_states = self.attn_stage.finish( + state.pop("hidden_states_after_attn"), state.forward_batch + ) + state.hidden_states_mlp_input = self.ffn_stage.prepare( + hidden_states, state.forward_batch ) def op_mlp(self, state): @@ -1674,16 +1671,13 @@ class Dots3DecoderLayer(nn.Module): state.hidden_states_mlp_output = hidden_states def op_comm_postprocess_layer(self, state): - hidden_states, residual = self.layer_communicator.postprocess_layer( - state.pop("hidden_states_mlp_output"), - state.pop("residual_after_comm_pre_mlp"), - state.forward_batch, + hidden_states = self.ffn_stage.postprocess( + state.pop("hidden_states_mlp_output"), state.forward_batch ) output = dict( positions=state.positions, hidden_states=hidden_states, - residual=residual, forward_batch=state.forward_batch, zero_allocator=state.zero_allocator, tbo_subbatch_index=state.tbo_subbatch_index, @@ -1766,12 +1760,12 @@ class Dots3Model(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) normal_start_layer = self.start_layer normal_end_layer = self.end_layer @@ -1787,38 +1781,28 @@ class Dots3Model(nn.Module): for i in range(normal_start_layer, normal_end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): layer = self.layers[i] - hidden_states, residual = layer( - positions, hidden_states, forward_batch, residual, zero_allocator + hidden_states = layer( + positions, hidden_states, forward_batch, zero_allocator ) if normal_end_layer != self.end_layer: - hidden_states, residual = model_forward_maybe_tbo( + hidden_states = model_forward_stages( layers=self.layers[normal_end_layer : self.end_layer], enable_tbo=True, positions=positions, forward_batch=forward_batch, hidden_states=hidden_states, - residual=residual, zero_allocator=zero_allocator, ) - last_layer = self.layers[self.end_layer - 1] - hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch - ) + hidden_states = residual_batch.finish(hidden_states, forward_batch) if not self.pp_group.is_last_rank: - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) + return residual_batch.to_pp(hidden_states, forward_batch) else: if not forward_batch.forward_mode.is_idle(): - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.norm + ) return hidden_states diff --git a/python/sglang/srt/models/dots3_common/nextn.py b/python/sglang/srt/models/dots3_common/nextn.py index c057321ff0..79da200858 100644 --- a/python/sglang/srt/models/dots3_common/nextn.py +++ b/python/sglang/srt/models/dots3_common/nextn.py @@ -114,20 +114,17 @@ class Dot3NoteModelNextN(nn.Module): ) ) - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) with get_global_expert_distribution_recorder().disable_this_region(): - hidden_states, residual = head.decoder( - positions, hidden_states, forward_batch, residual, zero_allocator + hidden_states = head.decoder( + positions, hidden_states, forward_batch, zero_allocator ) - hidden_states, residual = head.decoder.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch - ) + hidden_states = residual_batch.finish(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): - if residual is None: - hidden_states = head.shared_head.norm(hidden_states) - else: - hidden_states, _ = head.shared_head.norm(hidden_states, residual) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, head.shared_head.norm + ) return hidden_states def _embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py index 09086bf015..a79cb72d5f 100644 --- a/python/sglang/srt/models/falcon_h1.py +++ b/python/sglang/srt/models/falcon_h1.py @@ -195,7 +195,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module): rms_norm_eps=config.rms_norm_eps, activation=config.hidden_act, use_rms_norm=config.mamba_rms_norm, - # Like the attention's o_proj: prepare_mlp completes the sum of the + # Like the attention's o_proj: the FFN input completes the sum of the # two mixers' partial outputs. reduce_results=False, prefix=f"{prefix}.mixer", diff --git a/python/sglang/srt/models/gigachat35.py b/python/sglang/srt/models/gigachat35.py index 3147aa42f7..d1fe5398b9 100644 --- a/python/sglang/srt/models/gigachat35.py +++ b/python/sglang/srt/models/gigachat35.py @@ -24,6 +24,7 @@ from transformers import PretrainedConfig import sglang.srt.models.deepseek_v2 as deepseek_v2 from sglang.srt.configs.gigachat35 import GigaChat35Config from sglang.srt.layers.communicator import get_attn_tp_context +from sglang.srt.layers.communicator.output import OutputTransform from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm from sglang.srt.layers.linear import ColumnParallelLinear @@ -133,7 +134,7 @@ class GigaChat35PassthroughNorm(nn.Module): Used as the ``input_layernorm`` slot when a layer has no pre-norm (``layernorm_type="post"``); it only folds the residual so the - LayerCommunicator's prepare-step bookkeeping still works. + attention boundary receives the folded residual. """ def forward( @@ -153,10 +154,10 @@ class GigaChat35PassthroughNorm(nn.Module): class GigaChat35MlpPrepNorm(nn.Module): """Pre-MLP norm wrapper that folds the post-attention (sandwich) norm. - Replaces the LayerCommunicator's ``post_attention_layernorm`` slot so the + Serves as the FFN boundary's norm so the optional ``post_self_attn_layernorm`` (applied to the attention output, before the residual add) and the pre-MLP ``post_attention_layernorm`` are - both threaded through ``prepare_mlp`` -- matching the per-layer math of the + both run during FFN input preparation, matching the per-layer math of the ``pre_post`` sandwich exactly. """ @@ -389,7 +390,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer): ) if hasattr(self.self_attn, "out_proj"): self.self_attn.out_proj.reduce_results = False - self.layer_communicator.qkv_latent_func = None + qkv_latent_func = None else: self.self_attn = GigaChat35AttentionMLA( config=config, @@ -409,7 +410,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer): prefix=add_prefix("self_attn", prefix), alt_stream=alt_stream, ) - self.layer_communicator.qkv_latent_func = self.self_attn.prepare_qkv_latent + qkv_latent_func = self.self_attn.prepare_qkv_latent self.is_sparse = self.is_layer_sparse @@ -440,11 +441,17 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer): post_layernorm=self.post_self_attn_layernorm, ) # The communicator chooses its fused steps from its norms when built. - self.layer_communicator = self._build_layer_communicator( + self.attn_stage, self.ffn_stage = self._build_stages( input_layernorm=attn_prepare_layernorm, post_attention_layernorm=mlp_prepare_layernorm, - qkv_latent_func=self.layer_communicator.qkv_latent_func, - allow_deferred_ffn_reduction=False, + qkv_latent_func=qkv_latent_func, + output=( + OutputTransform( + self.post_feedforward_layernorm, before_reduce_scatter=True + ) + if self.post_feedforward_layernorm is not None + else None + ), ) def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool: @@ -461,13 +468,10 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], zero_allocator: BumpAllocator, **kwargs, - ) -> tuple[torch.Tensor, torch.Tensor]: - hidden_states, residual = self.layer_communicator.prepare_attn( - hidden_states, residual, forward_batch - ) + ) -> torch.Tensor: + hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) if self.use_linear_attn: hidden_states = self.self_attn( @@ -481,26 +485,22 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer): forward_batch=forward_batch, zero_allocator=zero_allocator, input_on_attention_tp_slices=( - self.layer_communicator.input_on_attention_tp_slices + self.attn_stage.input_on_attention_tp_slices ), ) get_attn_tp_context().clear_attn_inputs() - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) # Unlike deepseek_v2, no moe_output_buffer_ctx here: non-inplace MoE # runners then allocate their output per forward instead of recycling # the layer-input buffer. The default (inplace) runners are unaffected. - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + with self.ffn_stage.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) - if self.post_feedforward_layernorm is not None: - hidden_states = self.post_feedforward_layernorm(hidden_states) - - hidden_states, residual = ffn_exit.finish(hidden_states, residual) - return hidden_states, residual + hidden_states = ffn_exit.finish(hidden_states) + return hidden_states class GigaChat35Model(nn.Module): @@ -559,12 +559,12 @@ class GigaChat35Model(nn.Module): hidden_states = inputs_embeds else: hidden_states = self.embed_tokens(input_ids) - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) total_num_layers = self.end_layer - self.start_layer zero_allocator = BumpAllocator( @@ -574,33 +574,19 @@ class GigaChat35Model(nn.Module): ) for i in range(self.start_layer, self.end_layer): - hidden_states, residual = self.layers[i]( + hidden_states = self.layers[i]( positions=positions, hidden_states=hidden_states, forward_batch=forward_batch, - residual=residual, zero_allocator=zero_allocator, ) - hidden_states, residual = self.layers[ - self.end_layer - 1 - ].layer_communicator.finish_layer_stack( - hidden_states, - residual, - forward_batch, - preserve_declared=not self.pp_group.is_last_rank, - ) - if not self.pp_group.is_last_rank: - return PPProxyTensors( - {"hidden_states": hidden_states, "residual": residual} - ) + return residual_batch.to_pp(hidden_states, forward_batch) + hidden_states = residual_batch.finish(hidden_states, forward_batch) if hidden_states.shape[0] != 0: - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) return hidden_states diff --git a/python/sglang/srt/models/gigachat35_mtp.py b/python/sglang/srt/models/gigachat35_mtp.py index fd98c8f8f2..31dc0b037f 100644 --- a/python/sglang/srt/models/gigachat35_mtp.py +++ b/python/sglang/srt/models/gigachat35_mtp.py @@ -82,7 +82,7 @@ class GigaChat35ModelNextN(nn.Module): positions: torch.Tensor, forward_batch: ForwardBatch, input_embeds: Optional[torch.Tensor] = None, - ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + ) -> torch.Tensor: zero_allocator = BumpAllocator( buffer_size=2, dtype=torch.float32, @@ -107,28 +107,24 @@ class GigaChat35ModelNextN(nn.Module): ) ) - residual = residual_batch.start(forward_batch) - hidden_states, residual = self.decoder( + residual_batch.start(forward_batch) + hidden_states = self.decoder( positions=positions, hidden_states=hidden_states, forward_batch=forward_batch, - residual=residual, zero_allocator=zero_allocator, ) - hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch - ) + hidden_states = residual_batch.finish(hidden_states, forward_batch) hidden_states_before_norm = None if not forward_batch.forward_mode.is_idle(): - hidden_states_before_norm = self.decoder.layer_communicator.snapshot( - hidden_states, residual + hidden_states_before_norm = residual_batch.snapshot( + hidden_states, forward_batch + ) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.shared_head.norm ) - if residual is not None: - hidden_states, _ = self.shared_head.norm(hidden_states, residual) - else: - hidden_states = self.shared_head.norm(hidden_states) return hidden_states, hidden_states_before_norm diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 8bb5013648..07d83d1dd9 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -26,7 +26,7 @@ from transformers import PretrainedConfig from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz from sglang.srt.batch_overlap.single_batch_overlap import SboFlags -from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo +from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages from sglang.srt.distributed import ( get_pp_indices, ) @@ -40,9 +40,10 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.communicator import ( - LayerCommunicator, - LayerFacts, + declare_attn, + declare_ffn, enable_moe_dense_fully_dp, + make_stages, ) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( @@ -837,14 +838,6 @@ class Glm4MoeDecoderLayer(nn.Module): is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False) is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False) - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=1 if is_nextn else config.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - if self.is_layer_sparse: self.mlp = Glm4MoeSparseMoeBlock( config=config, @@ -873,11 +866,21 @@ class Glm4MoeDecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1, ) # Detect if QKV uses aiter FP8 per-token quant so we can fuse @@ -936,18 +939,14 @@ class Glm4MoeDecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], capture_output=None, ) -> torch.Tensor: - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - quant_format=self.attn_quant_format, - capture_output=capture_output, - ) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + quant_format=self.attn_quant_format, + capture_output=capture_output, ) hidden_states = self.self_attn( @@ -956,15 +955,14 @@ class Glm4MoeDecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + with self.ffn_stage.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) + hidden_states = ffn_exit.finish(hidden_states) - return hidden_states, residual + return hidden_states def op_comm_prepare_attn( self, @@ -972,16 +970,10 @@ class Glm4MoeDecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], tbo_subbatch_index: Optional[int] = None, ): - state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = ( - self.layer_communicator.prepare_attn( - hidden_states, - residual, - forward_batch, - quant_format=self.attn_quant_format, - ) + state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare( + hidden_states, forward_batch ) state.update( dict( @@ -992,25 +984,21 @@ class Glm4MoeDecoderLayer(nn.Module): ) def op_comm_prepare_mlp(self, state): - state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = ( - self.layer_communicator.prepare_mlp( - state.pop("hidden_states_after_attn"), - state.pop("residual_after_input_ln"), - state.forward_batch, - ) + hidden_states = self.attn_stage.finish( + state.pop("hidden_states_after_attn"), state.forward_batch + ) + state.hidden_states_mlp_input = self.ffn_stage.prepare( + hidden_states, state.forward_batch ) def op_comm_postprocess_layer(self, state): - hidden_states, residual = self.layer_communicator.postprocess_layer( - state.pop("hidden_states_mlp_output"), - state.pop("residual_after_comm_pre_mlp"), - state.forward_batch, + hidden_states = self.ffn_stage.postprocess( + state.pop("hidden_states_mlp_output"), state.forward_batch ) output = dict( positions=state.positions, hidden_states=hidden_states, - residual=residual, forward_batch=state.forward_batch, tbo_subbatch_index=state.tbo_subbatch_index, ) @@ -1090,12 +1078,12 @@ class Glm4MoeModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) normal_start_layer = self.start_layer normal_end_layer = self.end_layer @@ -1112,46 +1100,32 @@ class Glm4MoeModel(nn.Module): for i in range(normal_start_layer, normal_end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): layer = self.layers[i] - hidden_states, residual = layer( + hidden_states = layer( positions, hidden_states, forward_batch, - residual, capture_output=aux_hidden_states.capture if i in self.layers_to_capture else None, ) if normal_end_layer != self.end_layer: - hidden_states, residual = model_forward_maybe_tbo( + hidden_states = model_forward_stages( layers=self.layers[normal_end_layer : self.end_layer], enable_tbo=True, positions=positions, forward_batch=forward_batch, hidden_states=hidden_states, - residual=residual, ) - last_layer = self.layers[self.end_layer - 1] - hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, - residual, - forward_batch, - preserve_declared=not self.pp_group.is_last_rank, - ) if not self.pp_group.is_last_rank: - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) + return residual_batch.to_pp(hidden_states, forward_batch) else: + hidden_states = residual_batch.finish(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.norm + ) if len(aux_hidden_states) == 0: return hidden_states return hidden_states, aux_hidden_states diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 107ae88c98..bd9223ac67 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -24,7 +24,7 @@ from torch import nn from transformers import PretrainedConfig from sglang.srt.batch_overlap.single_batch_overlap import SboFlags -from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo +from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) @@ -34,10 +34,11 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.communicator import ( - LayerCommunicator, - LayerFacts, + declare_attn, + declare_ffn, enable_moe_dense_fully_dp, get_attn_tp_context, + make_stages, ) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( @@ -549,14 +550,6 @@ class Glm4MoeLiteDecoderLayer(nn.Module): is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False) is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False) - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=1 if is_nextn else config.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - if self.is_layer_sparse: self.mlp = Glm4MoeLiteSparseMoeBlock( config=config, @@ -588,12 +581,25 @@ class Glm4MoeLiteDecoderLayer(nn.Module): self._gfx95_quant_format = self._detect_gfx95_quant_format() - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, - qkv_latent_func=self.self_attn.prepare_qkv_latent, + self.attn_stage, self.ffn_stage = make_stages( + ( + declare_attn(), + self.input_layernorm, + {"qkv_latent_func": self.self_attn.prepare_qkv_latent}, + ), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1, ) def _detect_gfx95_quant_format(self) -> str: @@ -624,18 +630,14 @@ class Glm4MoeLiteDecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], zero_allocator: BumpAllocator, capture_output=None, ) -> torch.Tensor: - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - quant_format=getattr(self, "_gfx95_quant_format", ""), - capture_output=capture_output, - ) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + quant_format=getattr(self, "_gfx95_quant_format", ""), + capture_output=capture_output, ) hidden_states = self.self_attn( @@ -643,23 +645,20 @@ class Glm4MoeLiteDecoderLayer(nn.Module): hidden_states=hidden_states, forward_batch=forward_batch, zero_allocator=zero_allocator, - input_on_attention_tp_slices=( - self.layer_communicator.input_on_attention_tp_slices - ), + input_on_attention_tp_slices=(self.attn_stage.input_on_attention_tp_slices), ) if isinstance(hidden_states, tuple): hidden_states = hidden_states[0] get_attn_tp_context().clear_attn_inputs() - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + with self.ffn_stage.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) + hidden_states = ffn_exit.finish(hidden_states) - return hidden_states, residual + return hidden_states def op_comm_prepare_attn( self, @@ -667,12 +666,11 @@ class Glm4MoeLiteDecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], zero_allocator: BumpAllocator, tbo_subbatch_index: Optional[int] = None, ): - state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = ( - self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch) + state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare( + hidden_states, forward_batch ) if get_moe_a2a_backend().is_mori(): state.num_tokens = hidden_states.shape[0] @@ -686,25 +684,21 @@ class Glm4MoeLiteDecoderLayer(nn.Module): ) def op_comm_prepare_mlp(self, state): - state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = ( - self.layer_communicator.prepare_mlp( - state.pop("hidden_states_after_attn"), - state.pop("residual_after_input_ln"), - state.forward_batch, - ) + hidden_states = self.attn_stage.finish( + state.pop("hidden_states_after_attn"), state.forward_batch + ) + state.hidden_states_mlp_input = self.ffn_stage.prepare( + hidden_states, state.forward_batch ) def op_comm_postprocess_layer(self, state): - hidden_states, residual = self.layer_communicator.postprocess_layer( - state.pop("hidden_states_mlp_output"), - state.pop("residual_after_comm_pre_mlp"), - state.forward_batch, + hidden_states = self.ffn_stage.postprocess( + state.pop("hidden_states_mlp_output"), state.forward_batch ) output = dict( positions=state.positions, hidden_states=hidden_states, - residual=residual, forward_batch=state.forward_batch, zero_allocator=state.zero_allocator, tbo_subbatch_index=state.tbo_subbatch_index, @@ -782,12 +776,12 @@ class Glm4MoeLiteModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) device = hidden_states.device zero_allocator = BumpAllocator( buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1), @@ -809,11 +803,10 @@ class Glm4MoeLiteModel(nn.Module): for i in range(normal_start_layer, normal_end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): layer = self.layers[i] - hidden_states, residual = layer( + hidden_states = layer( positions, hidden_states, forward_batch, - residual, zero_allocator, capture_output=aux_hidden_states.capture if i in self.layers_to_capture @@ -821,36 +814,23 @@ class Glm4MoeLiteModel(nn.Module): ) if normal_end_layer != self.end_layer: - hidden_states, residual = model_forward_maybe_tbo( + hidden_states = model_forward_stages( layers=self.layers[normal_end_layer : self.end_layer], enable_tbo=True, positions=positions, forward_batch=forward_batch, hidden_states=hidden_states, - residual=residual, zero_allocator=zero_allocator, ) - last_layer = self.layers[self.end_layer - 1] - hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, - residual, - forward_batch, - preserve_declared=not self.pp_group.is_last_rank, - ) if not self.pp_group.is_last_rank: - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) + return residual_batch.to_pp(hidden_states, forward_batch) else: + hidden_states = residual_batch.finish(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.norm + ) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/glm4_moe_lite_nextn.py b/python/sglang/srt/models/glm4_moe_lite_nextn.py index 2aded48fe7..867a0dcfea 100644 --- a/python/sglang/srt/models/glm4_moe_lite_nextn.py +++ b/python/sglang/srt/models/glm4_moe_lite_nextn.py @@ -115,20 +115,17 @@ class Glm4MoeLiteModelNextN(nn.Module): ) ) - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) with get_global_expert_distribution_recorder().disable_this_region(): - hidden_states, residual = self.decoder( - positions, hidden_states, forward_batch, residual, zero_allocator + hidden_states = self.decoder( + positions, hidden_states, forward_batch, zero_allocator ) - hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch - ) + hidden_states = residual_batch.finish(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): - if residual is not None: - hidden_states, _ = self.shared_head.norm(hidden_states, residual) - else: - hidden_states = self.shared_head.norm(hidden_states) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.shared_head.norm + ) return hidden_states diff --git a/python/sglang/srt/models/glm4_moe_nextn.py b/python/sglang/srt/models/glm4_moe_nextn.py index f7f11f6511..a356943e9a 100644 --- a/python/sglang/srt/models/glm4_moe_nextn.py +++ b/python/sglang/srt/models/glm4_moe_nextn.py @@ -101,20 +101,15 @@ class Glm4MoeModelNextN(nn.Module): ) ) - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) with get_global_expert_distribution_recorder().disable_this_region(): - hidden_states, residual = self.decoder( - positions, hidden_states, forward_batch, residual - ) + hidden_states = self.decoder(positions, hidden_states, forward_batch) - hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch - ) + hidden_states = residual_batch.finish(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): - if residual is not None: - hidden_states, _ = self.shared_head.norm(hidden_states, residual) - else: - hidden_states = self.shared_head.norm(hidden_states) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.shared_head.norm + ) return hidden_states diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index b5d310c773..a5e4520e56 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -2,7 +2,7 @@ import logging from array import array from contextlib import nullcontext from functools import partial -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import Iterable, List, Optional, Tuple, Union import torch from torch import nn @@ -13,7 +13,7 @@ from sglang.kernels.ops.layernorm.mhc import hc_contract from sglang.kernels.ops.layernorm.mhc import hc_post as _hc_post_fn from sglang.kernels.ops.layernorm.mhc import hc_pre as _hc_pre_fn from sglang.srt.batch_overlap.two_batch_overlap import ( - model_forward_maybe_tbo, + model_forward_stages, ) from sglang.srt.configs.glm5_next import Glm5NextConfig, Glm5NextTextConfig from sglang.srt.configs.model_config import is_deepseek_dsa @@ -27,12 +27,15 @@ from sglang.srt.layers.attention import vision_utils from sglang.srt.layers.attention.vision import VisionAttention from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.communicator import ( - LayerCommunicator, - LayerFacts, - MHCLayerCommunicator, + PLAIN_RESIDUAL, + MHCState, + declare_attn, + declare_ffn, enable_moe_dense_fully_dp, get_attn_tp_context, + make_stages, ) +from sglang.srt.layers.communicator.residual import access as residual_access from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( @@ -727,14 +730,6 @@ class Glm5NextDecoderLayer(nn.Module): is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False) is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False) - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=1 if is_nextn else config.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - if self.is_layer_sparse: self.mlp = Glm5NextMoE( config=config, @@ -784,36 +779,52 @@ class Glm5NextDecoderLayer(nn.Module): torch.empty(mix_hc, hc_dim, dtype=torch.float32) ) - shared_kwargs: Dict[str, Any] = dict( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, - qkv_latent_func=( - self.self_attn.prepare_qkv_latent if not self.is_linear_attn else None - ), - ) - + terminal = layer_id == (1 if is_nextn else config.num_hidden_layers) - 1 + residual = PLAIN_RESIDUAL if self.config.mhc: - mhc_kwargs: Dict[str, Any] = dict( + residual = MHCState( hc_mult=config.hc_mult, hc_attn_pre=self.hc_attn_pre, hc_ffn_pre=self.hc_ffn_pre, hc_post=self.hc_post, - # Resolved once: env and platform are frozen after startup, - # and None keeps the dispatch off the per-boundary path. hc_ffn_post_pre=( self.hc_ffn_post_pre if is_cross_layer_mhc_fusion_enabled() else None ), + is_last_layer=terminal, + ).layer_residual() + self.attn_stage, self.ffn_stage = make_stages( + ( + declare_attn( + read=residual.attention_read, update=residual.attention_update + ), + self.input_layernorm, + { + "residual_in_hidden": residual.ffn_update.at_producer, + "qkv_latent_func": self.self_attn.prepare_qkv_latent + if not self.is_linear_attn + else None, + }, + ), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + read=residual.ffn_read, + update=residual.ffn_update, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, + next_sparse=self.is_layer_sparse, + update=residual.ffn_update, ) - self.layer_communicator = MHCLayerCommunicator( - **shared_kwargs, - **mhc_kwargs, - ) - else: - self.layer_communicator = LayerCommunicator(**shared_kwargs) + if layer_id != 0 + else None, + terminal=terminal, + ) def _hc_pre( self, hc_fn, hc_scale, hc_base, hidden_states, out_norm_weight, out_norm_eps @@ -915,21 +926,15 @@ class Glm5NextDecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], zero_allocator: Optional[BumpAllocator] = None, gemm_output_zero_allocator: BumpAllocator = None, prev_topk_indices: Optional[torch.Tensor] = None, capture_output=None, ): - hidden_states_orig = self.layer_communicator.buffer(hidden_states) + hidden_states_orig = residual_access.buffer(hidden_states) - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - capture_output=capture_output, - ) + hidden_states = self.attn_stage.prepare( + hidden_states, forward_batch, capture_output=capture_output ) hidden_states = self.self_attn( @@ -937,9 +942,7 @@ class Glm5NextDecoderLayer(nn.Module): hidden_states=hidden_states, forward_batch=forward_batch, zero_allocator=zero_allocator, - input_on_attention_tp_slices=( - self.layer_communicator.input_on_attention_tp_slices - ), + input_on_attention_tp_slices=(self.attn_stage.input_on_attention_tp_slices), prev_topk_indices=prev_topk_indices, ) if isinstance(hidden_states, tuple): @@ -948,11 +951,8 @@ class Glm5NextDecoderLayer(nn.Module): topk_indices = None get_attn_tp_context().clear_attn_inputs() - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, - residual, - forward_batch, - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) if isinstance(self.mlp, Glm5NextMLP): gemm_output_zero_allocator = None @@ -968,15 +968,15 @@ class Glm5NextDecoderLayer(nn.Module): else: _mlp_ctx = nullcontext() - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit, _mlp_ctx: + with self.ffn_stage.exit(forward_batch) as ffn_exit, _mlp_ctx: hidden_states = self.mlp( hidden_states, forward_batch, gemm_output_zero_allocator, ) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) + hidden_states = ffn_exit.finish(hidden_states) - return hidden_states, residual, topk_indices + return (hidden_states, topk_indices) class Glm5NextModel(nn.Module): @@ -1106,12 +1106,12 @@ class Glm5NextModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) device = hidden_states.device zero_allocator = BumpAllocator( buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1), @@ -1165,11 +1165,10 @@ class Glm5NextModel(nn.Module): aux_hidden_states.capture(aux_hidden_state, owned=owned) layer = self.layers[i] - hidden_states, residual, topk_indices = layer( + (hidden_states, topk_indices) = layer( positions, hidden_states, forward_batch, - residual, zero_allocator, gemm_output_zero_allocator, prev_topk_indices=topk_indices, @@ -1179,38 +1178,25 @@ class Glm5NextModel(nn.Module): ) if normal_end_layer != self.end_layer: - hidden_states, residual = model_forward_maybe_tbo( + hidden_states = model_forward_stages( layers=self.layers[normal_end_layer : self.end_layer], enable_tbo=True, positions=positions, forward_batch=forward_batch, hidden_states=hidden_states, - residual=residual, zero_allocator=zero_allocator, ) - last_layer = self.layers[self.end_layer - 1] - hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, - residual, - forward_batch, - preserve_declared=not self.pp_group.is_last_rank, - ) if not self.pp_group.is_last_rank: if self.config.mhc: return PPProxyTensors({"hidden_states": hidden_states}) - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) + return residual_batch.to_pp(hidden_states, forward_batch) else: + hidden_states = residual_batch.finish(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.norm + ) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index 4c24814f90..5f335d186f 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -14,7 +14,11 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, AuxHiddenStateList, ) -from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator import ( + declare_attn, + declare_ffn, + make_stages, +) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import GemmaRMSNorm @@ -421,19 +425,17 @@ class _InternS2MobiusDecoderMixin: def _forward_after_attention( self, hidden_states: torch.Tensor, - residual: torch.Tensor | None, forward_batch: ForwardBatch, meta_mlp: nn.ModuleList, - ) -> tuple[torch.Tensor, torch.Tensor | None]: - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + ) -> torch.Tensor: + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + with self.ffn_stage.exit(forward_batch) as ffn_exit: hidden_states = self._forward_mobius_mlp( hidden_states, forward_batch, meta_mlp ) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) - return hidden_states, residual + hidden_states = ffn_exit.finish(hidden_states) + return hidden_states class InternS2MobiusLayerMlp(nn.Module): @@ -481,13 +483,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module): layer_prefix = prefix.removesuffix(".linear_attn") self._init_mobius_mlp(config, quant_config, layer_prefix) - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=True, - is_previous_layer_sparse=True, - is_next_layer_sparse=True, - ) + self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = GemmaRMSNorm( config.hidden_size, eps=config.rms_norm_eps @@ -496,37 +492,43 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module): _enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz) ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, - enable_fused_ar_quant=enable_fused_ar_quant, - fused_ar_quant_keep_bf16=enable_fused_ar_quant, - allow_deferred_ffn_reduction=False, + self.attn_stage, self.ffn_stage = make_stages( + ( + declare_attn(), + self.input_layernorm, + { + "enable_fused_ar_quant": enable_fused_ar_quant, + "fused_ar_quant_keep_bf16": enable_fused_ar_quant, + }, + ), + ( + declare_ffn( + sparse=True, + next_sparse=True, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn(sparse=True, next_sparse=True) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) def forward( self, hidden_states: torch.Tensor, - residual: torch.Tensor | None, meta_mlp: nn.ModuleList, **kwargs, ): forward_batch = kwargs["forward_batch"] - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - captured_last_layer_outputs=kwargs.get("captured_last_layer_outputs"), - ) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + captured_last_layer_outputs=kwargs.get("captured_last_layer_outputs"), ) if not forward_batch.forward_mode.is_idle(): hidden_states = self.linear_attn(hidden_states, forward_batch) - return self._forward_after_attention( - hidden_states, residual, forward_batch, meta_mlp - ) + return self._forward_after_attention(hidden_states, forward_batch, meta_mlp) class InternS2MobiusAttentionDecoderLayer( @@ -614,13 +616,7 @@ class InternS2MobiusAttentionDecoderLayer( layer_prefix = prefix.removesuffix(".self_attn") self._init_mobius_mlp(config, quant_config, layer_prefix) - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=True, - is_previous_layer_sparse=True, - is_next_layer_sparse=True, - ) + self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = GemmaRMSNorm( config.hidden_size, eps=config.rms_norm_eps @@ -630,14 +626,26 @@ class InternS2MobiusAttentionDecoderLayer( enable_fused_ar_quant = ( _enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj) ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, - enable_fused_ar_quant=enable_fused_ar_quant, - fused_ar_quant_keep_bf16=False, - allow_deferred_ffn_reduction=False, + self.attn_stage, self.ffn_stage = make_stages( + ( + declare_attn(), + self.input_layernorm, + { + "enable_fused_ar_quant": enable_fused_ar_quant, + "fused_ar_quant_keep_bf16": False, + }, + ), + ( + declare_ffn( + sparse=True, + next_sparse=True, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn(sparse=True, next_sparse=True) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) self.alt_stream = alt_stream @@ -645,20 +653,16 @@ class InternS2MobiusAttentionDecoderLayer( self, positions: torch.Tensor, hidden_states: torch.Tensor, - residual: torch.Tensor | None, forward_batch: ForwardBatch, meta_mlp: nn.ModuleList, captured_last_layer_outputs: AuxHiddenStateAccumulator | None = None, **kwargs, ): del kwargs - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - captured_last_layer_outputs=captured_last_layer_outputs, - ) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + captured_last_layer_outputs=captured_last_layer_outputs, ) if not forward_batch.forward_mode.is_idle(): hidden_states = self.self_attention( @@ -666,9 +670,7 @@ class InternS2MobiusAttentionDecoderLayer( hidden_states=hidden_states, forward_batch=forward_batch, ) - return self._forward_after_attention( - hidden_states, residual, forward_batch, meta_mlp - ) + return self._forward_after_attention(hidden_states, forward_batch, meta_mlp) class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM): @@ -767,20 +769,17 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM): hidden_states = ( self.embed_tokens(input_ids) if input_embeds is None else input_embeds ) - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) aux_hidden_states = AuxHiddenStateList() for layer_idx, layer in enumerate(self.layers): - hidden_states, residual = layer( + hidden_states = layer( positions=positions, hidden_states=hidden_states, - residual=residual, forward_batch=forward_batch, meta_mlp=self.meta_mlp, - captured_last_layer_outputs=( - aux_hidden_states - if getattr(layer, "_is_layer_to_capture", False) - else None - ), + captured_last_layer_outputs=aux_hidden_states + if getattr(layer, "_is_layer_to_capture", False) + else None, ) if ( input_deepstack_embeds is not None @@ -788,19 +787,16 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM): and layer_idx < 3 ): start = self.hidden_size * layer_idx - hidden_states.add_( - input_deepstack_embeds[:, start : start + self.hidden_size] + hidden_states = residual_batch.add_to_output( + hidden_states, + forward_batch, + input_deepstack_embeds[:, start : start + self.hidden_size], ) - hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch - ) + hidden_states = residual_batch.finish(hidden_states, forward_batch) if hidden_states.shape[0] != 0: - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) return ( hidden_states if not aux_hidden_states diff --git a/python/sglang/srt/models/kimi_k25_eagle3.py b/python/sglang/srt/models/kimi_k25_eagle3.py index 7d6913d567..24b34b21fa 100644 --- a/python/sglang/srt/models/kimi_k25_eagle3.py +++ b/python/sglang/srt/models/kimi_k25_eagle3.py @@ -161,7 +161,7 @@ class Eagle3MLADecoderLayer(nn.Module): attn_input = torch.cat([embeds, hidden_states], dim=-1) # MLA's forward_absorb_prepare reads the qkv-down projection result - # from attn_tp_context. We bypass LayerCommunicator here (the eagle3 + # from attn_tp_context. We bypass layer boundary preparation here (the eagle3 # draft layer is one isolated layer with custom pre-attention norms), # so publish the attention input ourselves. get_attn_tp_context().set_attn_inputs( diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 000f0baf77..4ae33981f7 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -243,15 +243,14 @@ def _merge_weights_as_views( return merged, sizes -# K3 cannot use LayerCommunicator: the attn-res aggregation kernels replace -# input_layernorm / post_attention_layernorm, which the communicator expects -# to own. Instead the MLP/MoE modules gather/scatter around their own body: +# K3 manages its own boundaries: the attn-res aggregation kernels replace +# input_layernorm / post_attention_layernorm, and the MLP/MoE modules +# gather/scatter around their own body: # attention and the attn-res buffers stay in local (per-DP-rank) token space, # the MLP/MoE runs on the DP-gathered global batch, and the delayed prefix_sum # add stays local, applied after the scatter back. def _dp_local_buffer_group(): - """Symmetric-memory group for the local DP buffer (mirrors - CommunicateSummableTensorPairFn._scatter_hidden_states).""" + """Symmetric-memory group holding this DP rank's local token buffer.""" parallel = get_parallel() if parallel.tp_size == parallel.attn_dp_size: return get_parallel().tp_group @@ -2252,7 +2251,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA): self.o_proj.forward = _symm_o_proj_forward else: - # K3 has no LayerCommunicator, so o_proj must reduce within the + # K3 owns its output reduction, so o_proj must reduce within the # attn-TP group itself: the default full-TP collective is the wrong # group at attn_tp>1 and deadlocks idle DP ranks. self.o_proj.use_dp_attention_reduce = True @@ -2571,7 +2570,7 @@ class KimiK3DecoderLayer(nn.Module): ) -> torch.Tensor: # DP attention: idle ranks (padded to the global shape) have no # attention metadata; pass hidden_states through shape-preserving - # (same as the LayerCommunicator models' is_idle skip). + # (matching the idle skip in other DP-attention models). if forward_batch.forward_mode.is_idle(): return hidden_states @@ -2614,8 +2613,8 @@ class KimiK3DecoderLayer(nn.Module): forward_batch: ForwardBatch, zero_allocator: BumpAllocator, ) -> torch.Tensor: - # For MLA layers with q_lora_rank, set up communicator attn_inputs - # before the forward call (normally done by LayerCommunicator). + # For MLA layers with q_lora_rank, set up attn_inputs before the + # forward call (normally done by the attention boundary). from sglang.srt.layers.communicator import ( AttentionInputs, get_attn_tp_context, diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 62ba0fc8cb..cd4c576fef 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -21,8 +21,9 @@ from sglang.srt.environ import envs from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.communicator import ( - LayerCommunicator, - LayerFacts, + declare_attn, + declare_ffn, + make_stages, ) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( @@ -456,18 +457,21 @@ class LagunaDecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) def forward( @@ -475,16 +479,10 @@ class LagunaDecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], capture_output=None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - capture_output=capture_output, - ) + ) -> torch.Tensor: + hidden_states = self.attn_stage.prepare( + hidden_states, forward_batch, capture_output=capture_output ) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( @@ -492,17 +490,16 @@ class LagunaDecoderLayer(nn.Module): hidden_states=hidden_states, forward_batch=forward_batch, ) - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + with self.ffn_stage.exit(forward_batch) as ffn_exit: hidden_states = self.mlp( hidden_states, forward_batch=forward_batch, ) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) - return hidden_states, residual + hidden_states = ffn_exit.finish(hidden_states) + return hidden_states class LagunaModel(nn.Module): @@ -589,48 +586,37 @@ class LagunaModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) aux_hidden_states = AuxHiddenStateList() for i in range(self.start_layer, self.end_layer): layer = self.layers[i] - hidden_states, residual = layer( + hidden_states = layer( positions, hidden_states, forward_batch, - residual, capture_output=aux_hidden_states.capture if i in self.layers_to_capture else None, ) - last_layer = self.layers[self.end_layer - 1] - hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, - residual, - forward_batch, - preserve_declared=not self.pp_group.is_last_rank, - ) if not self.pp_group.is_last_rank: - return PPProxyTensors( - {"hidden_states": hidden_states, "residual": residual} - ) + return residual_batch.to_pp(hidden_states, forward_batch) + hidden_states = residual_batch.finish(hidden_states, forward_batch) if hidden_states.shape[0] != 0: - hidden_states = last_layer.layer_communicator.norm_output( + hidden_states = residual_batch.norm( hidden_states, - residual, + forward_batch, self.norm, - capture_output=( - aux_hidden_states.append - if self.end_layer in self.layers_to_capture - else None - ), + capture_output=aux_hidden_states.append + if self.end_layer in self.layers_to_capture + else None, ) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index eb0bf2f40f..e68af8a056 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -51,7 +51,11 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator import ( + declare_attn, + declare_ffn, + make_stages, +) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -66,7 +70,10 @@ from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import StandardTopKOutput, TopK -from sglang.srt.layers.moe.utils import filter_moe_weight_param_global_expert +from sglang.srt.layers.moe.utils import ( + filter_moe_weight_param_global_expert, + should_skip_mlp_all_reduce, +) from sglang.srt.layers.n_gram_embedding import NgramEmbedding from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.fp8_utils import ( @@ -323,7 +330,11 @@ class LongcatFlashMoE(nn.Module): # expert outputs across EP ranks, so an extra all-reduce double-counts. from sglang.srt.layers.moe.utils import get_moe_a2a_backend as _lc_gab - if self.tp_size > 1 and _lc_gab().is_none(): + if ( + self.tp_size > 1 + and _lc_gab().is_none() + and not should_skip_mlp_all_reduce() + ): final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_dim) @@ -420,41 +431,36 @@ class LongcatFlashDecoderLayer(nn.Module): self.attn_tp_size = get_parallel().attn_tp_size self.attn_tp_rank = get_parallel().attn_tp_rank - self.mlp_layer_facts = [ - LayerFacts.init_new( - layer_id=self.layer_id * 2 + i, - num_layers=config.num_hidden_layers, - is_layer_sparse=False, - is_previous_layer_sparse=False, - # TODO: Check if the following is correct. - is_next_layer_sparse=False, - ) - for i in range(2) - ] - self.mlp_layer_communicator = [ - LayerCommunicator( - layer_facts=self.mlp_layer_facts[i], - input_layernorm=self.input_layernorm[i], - post_attention_layernorm=self.post_attention_layernorm[i], - qkv_latent_func=self.self_attn[i].prepare_qkv_latent, - allow_deferred_ffn_reduction=False, - ) - for i in range(2) - ] - - self.moe_layer_facts = LayerFacts.init_new( - layer_id=self.layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=True, - is_previous_layer_sparse=True, - # TODO: Check if the following is correct. - is_next_layer_sparse=True, + self.attn_stage, self.moe_stage = make_stages( + ( + declare_attn(), + self.input_layernorm[0], + {"qkv_latent_func": self.self_attn[0].prepare_qkv_latent}, + ), + ( + declare_ffn(sparse=True, next_sparse=True), + self.post_attention_layernorm[0], + ), + previous=declare_ffn(sparse=True, next_sparse=True) + if self.layer_id != 0 + else None, + terminal=self.layer_id == config.num_hidden_layers - 1, ) - self.moe_layer_communicator = LayerCommunicator( - layer_facts=self.moe_layer_facts, - input_layernorm=self.input_layernorm[0], - post_attention_layernorm=self.post_attention_layernorm[0], - qkv_latent_func=self.self_attn[0].prepare_qkv_latent, + self.first_ffn, self.second_attn, self.second_ffn = make_stages( + ( + declare_ffn(ordinary_only=True), + self.post_attention_layernorm[0], + ), + ( + declare_attn(ordinary_only=True), + self.input_layernorm[1], + {"qkv_latent_func": self.self_attn[1].prepare_qkv_latent}, + ), + ( + declare_ffn(ordinary_only=True), + self.post_attention_layernorm[1], + ), + prepared_from=self.moe_stage.declaration, ) def forward( @@ -462,14 +468,11 @@ class LongcatFlashDecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], zero_allocator: BumpAllocator, prev_topk_indices: Optional[torch.Tensor], ) -> torch.Tensor: # first_attn - hidden_states, residual = self.moe_layer_communicator.prepare_attn( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: attn_out = self.self_attn[0]( positions=positions, @@ -484,52 +487,46 @@ class LongcatFlashDecoderLayer(nn.Module): hidden_states = attn_out # moe - hidden_states, residual = self.moe_layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) - moe_hidden_states = self.moe_layer_communicator.branch_output( + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.moe_stage.prepare(hidden_states, forward_batch) + moe_hidden_states = self.moe_stage.branch_output( self.mlp(hidden_states.clone()), forward_batch ) - hidden_states, residual, prev_topk_indices = self.forward_mlp( + hidden_states, prev_topk_indices = self.forward_mlp( hidden_states, positions, - residual, forward_batch, zero_allocator, prev_topk_indices, ) - hidden_states, residual = self.moe_layer_communicator.merge_branch( + hidden_states = self.moe_stage.merge_branch( moe_hidden_states, hidden_states, - residual, - self.mlp_layer_communicator[1], + self.second_ffn, forward_batch, ) - return hidden_states, residual, prev_topk_indices + return hidden_states, prev_topk_indices def forward_mlp( self, hidden_states, positions, - residual, forward_batch, zero_allocator, prev_topk_indices, ): # first_mlp, on the input the MoE's boundary read - hidden_states, residual = self.mlp_layer_communicator[0].branch_input( - self.moe_layer_communicator, hidden_states, residual, forward_batch + hidden_states = self.first_ffn.branch_input( + self.moe_stage, hidden_states, forward_batch ) - with self.mlp_layer_communicator[0].ffn_exit(forward_batch) as ffn_exit: + with self.first_ffn.exit(forward_batch) as ffn_exit: hidden_states = self.mlps[0](hidden_states) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) + hidden_states = ffn_exit.finish(hidden_states) # second_attn - hidden_states, residual = self.mlp_layer_communicator[1].prepare_attn( - hidden_states, residual, forward_batch - ) + hidden_states = self.second_attn.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: attn_out = self.self_attn[1]( positions=positions, @@ -544,14 +541,13 @@ class LongcatFlashDecoderLayer(nn.Module): hidden_states = attn_out # second_mlp - hidden_states, residual = self.mlp_layer_communicator[1].prepare_mlp( - hidden_states, residual, forward_batch - ) - with self.mlp_layer_communicator[1].ffn_exit(forward_batch) as ffn_exit: + hidden_states = self.second_attn.finish(hidden_states, forward_batch) + hidden_states = self.second_ffn.prepare(hidden_states, forward_batch) + with self.second_ffn.exit(forward_batch) as ffn_exit: hidden_states = self.mlps[1](hidden_states) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) + hidden_states = ffn_exit.finish(hidden_states) - return hidden_states, residual, prev_topk_indices + return hidden_states, prev_topk_indices class LongcatFlashModel(nn.Module): @@ -625,39 +621,29 @@ class LongcatFlashModel(nn.Module): else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) aux_hidden_states = [] topk_indices = None for i in range(total_num_layers): if i in self.layers_to_capture: aux_hidden_states.append( - self.layers[i].moe_layer_communicator.snapshot( - hidden_states, residual, at_input=True - ) + self.layers[i].attn_stage.snapshot(hidden_states, forward_batch) ) with get_global_expert_distribution_recorder().with_current_layer(i): layer = self.layers[i] - hidden_states, residual, topk_indices = layer( + hidden_states, topk_indices = layer( positions, hidden_states, forward_batch, - residual, zero_allocator, topk_indices, ) - last_layer = self.layers[-1] - hidden_states, residual = last_layer.moe_layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.norm, skip_empty=True ) - if hidden_states.shape[0] != 0: - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) - if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/longcat_flash_nextn.py b/python/sglang/srt/models/longcat_flash_nextn.py index 373e223e57..3b2e642a76 100644 --- a/python/sglang/srt/models/longcat_flash_nextn.py +++ b/python/sglang/srt/models/longcat_flash_nextn.py @@ -41,7 +41,11 @@ from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz from sglang.srt.configs import LongcatFlashConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers import deep_gemm_wrapper -from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator import ( + declare_attn, + declare_ffn, + make_stages, +) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -160,17 +164,15 @@ class LongcatFlashDenseDecoderLayer(nn.Module): self.attn_tp_size = get_parallel().attn_tp_size self.attn_tp_rank = get_parallel().attn_tp_rank - self.layer_facts = LayerFacts.init_new( - layer_id=self.layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=False, - is_previous_layer_sparse=False, - is_next_layer_sparse=False, - ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, + + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn(sparse=False, next_sparse=False), + self.post_attention_layernorm, + ), + previous=declare_ffn() if self.layer_id != 0 else None, + terminal=self.layer_id == config.num_hidden_layers - 1, ) def forward( @@ -178,13 +180,10 @@ class LongcatFlashDenseDecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], zero_allocator: BumpAllocator, ) -> torch.Tensor: - hidden_states, residual = self.layer_communicator.prepare_attn( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( positions=positions, @@ -193,14 +192,11 @@ class LongcatFlashDenseDecoderLayer(nn.Module): zero_allocator=zero_allocator, ) - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) hidden_states = self.mlp(hidden_states) - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) - return hidden_states, residual + hidden_states = self.ffn_stage.postprocess(hidden_states, forward_batch) + return hidden_states class LongcatFlashModelNextN(nn.Module): @@ -270,21 +266,18 @@ class LongcatFlashModelNextN(nn.Module): ) ) - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) with get_global_expert_distribution_recorder().disable_this_region(): - hidden_states, residual = self.decoder( - positions, hidden_states, forward_batch, residual, zero_allocator + hidden_states = self.decoder( + positions, hidden_states, forward_batch, zero_allocator ) - hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch - ) + hidden_states = residual_batch.finish(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): - if residual is not None: - hidden_states, _ = self.final_layernorm(hidden_states, residual) - else: - hidden_states = self.final_layernorm(hidden_states) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.final_layernorm + ) return hidden_states diff --git a/python/sglang/srt/models/mellum.py b/python/sglang/srt/models/mellum.py index 8bbf656978..1915c728ba 100644 --- a/python/sglang/srt/models/mellum.py +++ b/python/sglang/srt/models/mellum.py @@ -25,7 +25,11 @@ import torch from torch import nn from transformers import PretrainedConfig -from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator import ( + declare_attn, + declare_ffn, + make_stages, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -442,22 +446,24 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer): is_previous_layer_sparse = _is_sparse(layer_id - 1) is_next_layer_sparse = _is_sparse(layer_id + 1) - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=cfg.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - self.input_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps) self.post_attention_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == cfg.num_hidden_layers - 1, ) diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 353355d40a..dd706f142d 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -22,7 +22,7 @@ import torch import torch.nn.functional as F from torch import nn -from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo +from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation @@ -33,9 +33,10 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStatePacker, ) from sglang.srt.layers.communicator import ( - LayerCommunicator, - LayerFacts, + declare_attn, + declare_ffn, enable_moe_dense_fully_dp, + make_stages, ) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( @@ -857,18 +858,21 @@ class MiMoV2DecoderLayer(nn.Module): config.hidden_size, eps=config.layernorm_epsilon ) - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) def forward( @@ -876,17 +880,13 @@ class MiMoV2DecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> torch.Tensor: # Self Attention - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - captured_last_layer_outputs=captured_last_layer_outputs, - ) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + captured_last_layer_outputs=captured_last_layer_outputs, ) if hidden_states.shape[0] != 0: @@ -896,15 +896,14 @@ class MiMoV2DecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + with self.ffn_stage.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) + hidden_states = ffn_exit.finish(hidden_states) - return hidden_states, residual + return hidden_states def is_moe_layer(self, layer_idx: int) -> bool: return ( @@ -923,11 +922,10 @@ class MiMoV2DecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], tbo_subbatch_index: Optional[int] = None, ): - state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = ( - self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch) + state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare( + hidden_states, forward_batch ) state.update( dict( @@ -938,25 +936,21 @@ class MiMoV2DecoderLayer(nn.Module): ) def op_comm_prepare_mlp(self, state): - state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = ( - self.layer_communicator.prepare_mlp( - state.pop("hidden_states_after_attn"), - state.pop("residual_after_input_ln"), - state.forward_batch, - ) + hidden_states = self.attn_stage.finish( + state.pop("hidden_states_after_attn"), state.forward_batch + ) + state.hidden_states_mlp_input = self.ffn_stage.prepare( + hidden_states, state.forward_batch ) def op_comm_postprocess_layer(self, state): - hidden_states, residual = self.layer_communicator.postprocess_layer( - state.pop("hidden_states_mlp_output"), - state.pop("residual_after_comm_pre_mlp"), - state.forward_batch, + hidden_states = self.ffn_stage.postprocess( + state.pop("hidden_states_mlp_output"), state.forward_batch ) output = dict( positions=state.positions, hidden_states=hidden_states, - residual=residual, forward_batch=state.forward_batch, tbo_subbatch_index=state.tbo_subbatch_index, ) @@ -1038,12 +1032,12 @@ class MiMoV2Model(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) aux_hidden_states = AuxHiddenStatePacker(len(self.layers_to_capture)) if forward_batch.can_run_tbo and not self.layers_to_capture: @@ -1053,39 +1047,29 @@ class MiMoV2Model(nn.Module): # skip first layer for TBO when starting from layer 0 if self.start_layer == 0: layer = self.layers[0] - hidden_states, residual = layer( - positions, hidden_states, forward_batch, residual - ) + hidden_states = layer(positions, hidden_states, forward_batch) tbo_start_layer = tbo_start_layer + 1 - hidden_states, residual = model_forward_maybe_tbo( + hidden_states = model_forward_stages( layers=self.layers[tbo_start_layer:tbo_end_layer], enable_tbo=True, positions=positions, forward_batch=forward_batch, hidden_states=hidden_states, - residual=residual, ) else: for i in range(self.start_layer, self.end_layer): layer = self.layers[i] - hidden_states, residual = layer( + hidden_states = layer( positions, hidden_states, forward_batch, - residual, - captured_last_layer_outputs=( - aux_hidden_states if i in self.layers_to_capture else None - ), + captured_last_layer_outputs=aux_hidden_states + if i in self.layers_to_capture + else None, ) - last_layer = self.layers[self.end_layer - 1] - hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, - residual, - forward_batch, - preserve_declared=not self.pp_group.is_last_rank, - ) + hidden_states = residual_batch.finish(hidden_states, forward_batch) # A draft targeting the final layer ("after layer # num_hidden_layers-1") maps to capture index num_hidden_layers, @@ -1095,27 +1079,21 @@ class MiMoV2Model(nn.Module): and self.config.num_hidden_layers in self.layers_to_capture ): aux_hidden_states.append( - last_layer.layer_communicator.snapshot(hidden_states, residual) + residual_batch.snapshot(hidden_states, forward_batch) ) hidden_states_before_norm = None if not self.pp_group.is_last_rank: - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) + return residual_batch.to_pp(hidden_states, forward_batch) else: if hidden_states.shape[0] > 0: if forward_batch.return_hidden_states_before_norm: - hidden_states_before_norm = last_layer.layer_communicator.snapshot( - hidden_states, residual + hidden_states_before_norm = residual_batch.snapshot( + hidden_states, forward_batch ) - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.norm + ) if len(aux_hidden_states) == 0: return hidden_states, hidden_states_before_norm diff --git a/python/sglang/srt/models/mimo_v2_nextn.py b/python/sglang/srt/models/mimo_v2_nextn.py index d0a638ffea..02ee8535fb 100644 --- a/python/sglang/srt/models/mimo_v2_nextn.py +++ b/python/sglang/srt/models/mimo_v2_nextn.py @@ -22,9 +22,10 @@ from transformers import PretrainedConfig from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers.communicator import ( - LayerCommunicator, - LayerFacts, + declare_attn, + declare_ffn, enable_moe_dense_fully_dp, + make_stages, ) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( @@ -117,18 +118,22 @@ class MiMoV2MTPLayer(nn.Module): self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.layernorm_epsilon ) - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=1, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_deferred_ffn_reduction=False, + + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == 1 - 1, ) def forward( @@ -136,12 +141,9 @@ class MiMoV2MTPLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> torch.Tensor: - hidden_states, residual = self.layer_communicator.prepare_attn( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( @@ -150,17 +152,16 @@ class MiMoV2MTPLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) with ( - self.layer_communicator.ffn_exit(forward_batch) as ffn_exit, + self.ffn_stage.exit(forward_batch) as ffn_exit, get_global_expert_distribution_recorder().disable_this_region(), ): hidden_states = self.mlp(hidden_states) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) + hidden_states = ffn_exit.finish(hidden_states) - return hidden_states, residual + return hidden_states class MiMoV2ModelNextN(nn.Module): @@ -220,26 +221,23 @@ class MiMoV2ModelNextN(nn.Module): dim=-1, ) ) - hidden_states, residual = self.mtp_block( + residual_batch.start(forward_batch) + hidden_states = self.mtp_block( positions=positions, hidden_states=hidden_states, forward_batch=forward_batch, - residual=residual_batch.start(forward_batch), ) - hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch - ) + hidden_states = residual_batch.finish(hidden_states, forward_batch) hidden_states_before_norm = None if not forward_batch.forward_mode.is_idle(): if forward_batch.return_hidden_states_before_norm: - hidden_states_before_norm = self.mtp_block.layer_communicator.snapshot( - hidden_states, residual + hidden_states_before_norm = residual_batch.snapshot( + hidden_states, forward_batch ) - if residual is not None: - hidden_states, _ = self.final_layernorm(hidden_states, residual) - else: - hidden_states = self.final_layernorm(hidden_states) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.final_layernorm + ) return hidden_states, hidden_states_before_norm diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index 67fe8d0aa5..9414955ed4 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -27,7 +27,7 @@ from torch import nn from transformers import PretrainedConfig from sglang.kernels.kernel_api_logging import debug_kernel_api -from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo +from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.aux_hidden_states import ( @@ -35,8 +35,9 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateList, ) from sglang.srt.layers.communicator import ( - LayerCommunicator, - LayerFacts, + declare_attn, + declare_ffn, + make_stages, ) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( @@ -980,19 +981,22 @@ class MiniMaxM2DecoderLayer(nn.Module): is_previous_layer_sparse = True is_next_layer_sparse = True - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) def forward( @@ -1000,17 +1004,13 @@ class MiniMaxM2DecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, ) -> torch.Tensor: # Self Attention - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - captured_last_layer_outputs=captured_last_layer_outputs, - ) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + captured_last_layer_outputs=captured_last_layer_outputs, ) if not forward_batch.forward_mode.is_idle(): hidden_states = self.self_attn( @@ -1021,15 +1021,14 @@ class MiniMaxM2DecoderLayer(nn.Module): # Fully Connected (MLP or MoE) - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + with self.ffn_stage.exit(forward_batch) as ffn_exit: hidden_states = self.block_sparse_moe(hidden_states, forward_batch) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) + hidden_states = ffn_exit.finish(hidden_states) - return hidden_states, residual + return hidden_states # TBO Operations for MiniMax Decoder Layer def op_comm_prepare_attn( @@ -1038,13 +1037,12 @@ class MiniMaxM2DecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], zero_allocator: BumpAllocator, tbo_subbatch_index: Optional[int] = None, ): """Communication prepare for attention - TBO operation""" - state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = ( - self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch) + state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare( + hidden_states, forward_batch ) state.update( dict( @@ -1057,26 +1055,22 @@ class MiniMaxM2DecoderLayer(nn.Module): def op_comm_prepare_mlp(self, state): """Communication prepare for MLP - TBO operation""" - state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = ( - self.layer_communicator.prepare_mlp( - state.pop("hidden_states_after_attn"), - state.pop("residual_after_input_ln"), - state.forward_batch, - ) + hidden_states = self.attn_stage.finish( + state.pop("hidden_states_after_attn"), state.forward_batch + ) + state.hidden_states_mlp_input = self.ffn_stage.prepare( + hidden_states, state.forward_batch ) def op_comm_postprocess_layer(self, state): """Communication postprocess for layer - TBO operation""" - hidden_states, residual = self.layer_communicator.postprocess_layer( - state.pop("hidden_states_mlp_output"), - state.pop("residual_after_comm_pre_mlp"), - state.forward_batch, + hidden_states = self.ffn_stage.postprocess( + state.pop("hidden_states_mlp_output"), state.forward_batch ) output = dict( positions=state.positions, hidden_states=hidden_states, - residual=residual, forward_batch=state.forward_batch, zero_allocator=state.zero_allocator, tbo_subbatch_index=state.tbo_subbatch_index, @@ -1146,22 +1140,21 @@ class MiniMaxM2Model(nn.Module): hidden_states = self.get_input_embeddings(input_ids) else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) aux_hidden_states = AuxHiddenStateList() if forward_batch.can_run_tbo: - hidden_states, residual = model_forward_maybe_tbo( + hidden_states = model_forward_stages( layers=self.layers, enable_tbo=True, positions=positions, forward_batch=forward_batch, hidden_states=hidden_states, - residual=residual, ) else: for i in range(self.start_layer, self.end_layer): @@ -1172,33 +1165,21 @@ class MiniMaxM2Model(nn.Module): ) with ctx: layer = self.layers[i] - hidden_states, residual = layer( + hidden_states = layer( positions=positions, forward_batch=forward_batch, hidden_states=hidden_states, - residual=residual, - captured_last_layer_outputs=( - aux_hidden_states if i in self.layers_to_capture else None - ), + captured_last_layer_outputs=aux_hidden_states + if i in self.layers_to_capture + else None, ) - last_layer = self.layers[self.end_layer - 1] - hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, - residual, - forward_batch, - preserve_declared=not self.pp_group.is_last_rank, - ) if not self.pp_group.is_last_rank: - return PPProxyTensors( - {"hidden_states": hidden_states, "residual": residual} - ) + return residual_batch.to_pp(hidden_states, forward_batch) + hidden_states = residual_batch.finish(hidden_states, forward_batch) if hidden_states.shape[0] != 0: - if residual is not None: - hidden_states, _ = self.norm(hidden_states, residual) - else: - hidden_states = self.norm(hidden_states) + hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 08c01dd606..7f541a56c3 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -23,7 +23,7 @@ import torch from torch import nn from transformers import PretrainedConfig -from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo +from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages from sglang.srt.configs.model_config import ( get_minimax_sparse_disable_value_layer_ids, get_minimax_sparse_layer_ids, @@ -40,9 +40,10 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateList, ) from sglang.srt.layers.communicator import ( - LayerCommunicator, - LayerFacts, + declare_attn, + declare_ffn, enable_moe_dense_fully_dp, + make_stages, ) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( @@ -1302,7 +1303,7 @@ class MiniMaxM3DecoderLayer(nn.Module): moe_layer_freq = getattr(config, "moe_layer_freq", None) # Means "MLP is a sparse MoE", not attention sparsity. Kept as ``is_layer_sparse`` - # because LayerCommunicator / LayerFacts / other models read this attr. + # because model construction and downstream integrations read this attr. self.is_layer_sparse = ( moe_layer_freq[layer_id] != 0 if moe_layer_freq is not None else True ) @@ -1352,19 +1353,22 @@ class MiniMaxM3DecoderLayer(nn.Module): is_previous_layer_sparse = _is_layer_sparse(layer_id - 1) is_next_layer_sparse = _is_layer_sparse(layer_id + 1) - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) def forward( @@ -1372,18 +1376,14 @@ class MiniMaxM3DecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, **kwargs, ) -> torch.Tensor: - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - captured_last_layer_outputs=captured_last_layer_outputs, - **kwargs, - ) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + captured_last_layer_outputs=captured_last_layer_outputs, + **kwargs, ) if hidden_states.shape[0] != 0: @@ -1393,11 +1393,10 @@ class MiniMaxM3DecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + with self.ffn_stage.exit(forward_batch) as ffn_exit: if self.is_layer_sparse or hidden_states.shape[0] != 0: hidden_states = self.mlp( hidden_states, @@ -1405,7 +1404,7 @@ class MiniMaxM3DecoderLayer(nn.Module): should_allreduce_fusion=ffn_exit.fuse_mlp_allreduce, use_reduce_scatter=ffn_exit.mlp_reduce_scatter, ) - return ffn_exit.finish(hidden_states, residual) + return ffn_exit.finish(hidden_states) class MiniMaxM3Model(nn.Module): @@ -1481,22 +1480,21 @@ class MiniMaxM3Model(nn.Module): hidden_states = embeds(input_ids) else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) aux_hidden_states = AuxHiddenStateList() if forward_batch.can_run_tbo: - hidden_states, residual = model_forward_maybe_tbo( + hidden_states = model_forward_stages( layers=self.layers, enable_tbo=True, positions=positions, forward_batch=forward_batch, hidden_states=hidden_states, - residual=residual, ) else: for i in range(self.start_layer, self.end_layer): @@ -1508,34 +1506,20 @@ class MiniMaxM3Model(nn.Module): ) with ctx: layer = self.layers[i] - hidden_states, residual = layer( + hidden_states = layer( positions=positions, forward_batch=forward_batch, hidden_states=hidden_states, - residual=residual, - captured_last_layer_outputs=( - aux_hidden_states - if getattr(layer, "_is_layer_to_capture", False) - else None - ), + captured_last_layer_outputs=aux_hidden_states + if getattr(layer, "_is_layer_to_capture", False) + else None, ) - last_layer = self.layers[self.end_layer - 1] - hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, - residual, - forward_batch, - preserve_declared=not self.pp_group.is_last_rank, - ) if not self.pp_group.is_last_rank: - return PPProxyTensors( - {"hidden_states": hidden_states, "residual": residual} - ) + return residual_batch.to_pp(hidden_states, forward_batch) + hidden_states = residual_batch.finish(hidden_states, forward_batch) if hidden_states.shape[0] != 0: - if residual is not None: - hidden_states, _ = self.norm(hidden_states, residual) - else: - hidden_states = self.norm(hidden_states) + hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/moss_vl.py b/python/sglang/srt/models/moss_vl.py index 4d6ae2cea1..91a7d83ccb 100644 --- a/python/sglang/srt/models/moss_vl.py +++ b/python/sglang/srt/models/moss_vl.py @@ -22,7 +22,11 @@ from sglang.srt.layers.attention.vision import ( VisionAttentionMetadata, prepare_vision_attention_metadata, ) -from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator import ( + declare_attn, + declare_ffn, + make_stages, +) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.conv import Conv3dLayer from sglang.srt.layers.layernorm import RMSNorm @@ -1012,18 +1016,15 @@ class MossVLSelfAttentionDecoderLayer(nn.Module): self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs ) - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=False, - is_previous_layer_sparse=False, - is_next_layer_sparse=False, - ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_deferred_ffn_reduction=False, + + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn(sparse=False, next_sparse=False), + self.post_attention_layernorm, + ), + previous=declare_ffn() if layer_id != 0 else None, + terminal=layer_id == config.num_hidden_layers - 1, ) def forward( @@ -1031,12 +1032,9 @@ class MossVLSelfAttentionDecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> torch.Tensor: # Self Attention - hidden_states, residual = self.layer_communicator.prepare_attn( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( positions=positions, @@ -1045,15 +1043,12 @@ class MossVLSelfAttentionDecoderLayer(nn.Module): ) # MLP - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, - residual, - forward_batch, - ) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + with self.ffn_stage.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) - return hidden_states, residual + hidden_states = ffn_exit.finish(hidden_states) + return hidden_states # ==================== Text Model ==================== @@ -1110,14 +1105,12 @@ class MossVLTextModel(nn.Module): vision_position_ids: Optional[torch.Tensor] = None, ) -> torch.Tensor: hidden_states = self.embed_tokens(input_ids) - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) for decoder_layer in self.layers: if isinstance(decoder_layer, MossVLCrossAttentionDecoderLayer): if not skip_cross_attention: - hidden_states, residual = LayerCommunicator.fold( - hidden_states, residual - ) + hidden_states = residual_batch.fold(hidden_states, forward_batch) hidden_states = decoder_layer( hidden_states=hidden_states, cross_attention_states=cross_attention_states, @@ -1127,27 +1120,17 @@ class MossVLTextModel(nn.Module): positions=positions, vision_position_ids=vision_position_ids, ) - hidden_states, residual = LayerCommunicator.written( - hidden_states, forward_batch - ) + hidden_states = residual_batch.written(hidden_states, forward_batch) elif isinstance(decoder_layer, MossVLSelfAttentionDecoderLayer): - hidden_states, residual = decoder_layer( + hidden_states = decoder_layer( positions=positions, hidden_states=hidden_states, forward_batch=forward_batch, - residual=residual, ) else: raise ValueError(f"Unknown decoder layer type {type(decoder_layer)}") - hidden_states, residual = LayerCommunicator.finish_layer_stack( - hidden_states, residual, forward_batch - ) - - if residual is not None: - hidden_states, _ = self.norm(hidden_states, residual) - else: - hidden_states = self.norm(hidden_states) + hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) return hidden_states diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index a6eba431c9..5adcd93367 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -738,7 +738,7 @@ class NemotronHModel(nn.Module): for i in range(self.start_layer, self.end_layer): if i in self.layers_to_capture: hidden_states, snapshot = self.layers[i].stage.capture_output( - hidden_states, forward_batch, at_input=True + hidden_states, forward_batch ) aux_hidden_states.append(snapshot) layer = self.layers[i] diff --git a/python/sglang/srt/models/nemotron_h_utils.py b/python/sglang/srt/models/nemotron_h_utils.py index b06c9ac108..8516a4bc80 100644 --- a/python/sglang/srt/models/nemotron_h_utils.py +++ b/python/sglang/srt/models/nemotron_h_utils.py @@ -4,22 +4,12 @@ from typing import Optional from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE from sglang.srt.layers.communicator import ( - NORM_QUANT_READ, - LayerCommunicator, - LayerFacts, - LayerStage, - Layout, - StageDecl, - StageInput, StageKind, - StageOutput, - SumGroup, - TokenAxis, - stage_edges, - token_axis_sizes, + declare_attn, + declare_ffn, + make_stages, ) from sglang.srt.layers.layernorm import RMSNorm -from sglang.srt.layers.moe.utils import get_moe_a2a_backend ATTN_LAYERS = (MAMBA, ATTENTION) @@ -37,87 +27,35 @@ def _stage_kind(pattern: str, layer_idx: int) -> Optional[StageKind]: return StageKind.ATTENTION if is_attn_layer(pattern[layer_idx]) else StageKind.FFN -def _stage_decl(pattern: str, layer_idx: int) -> StageDecl: - """What the stage at ``layer_idx`` declares. A mixer reads its input as - an attention does, computes on the attention's rows and leaves its - attention-TP sum to an FFN stage after it, or, when a fused kernel takes - it, to a mixer after it. An FFN computes on the TP group's rows, a MoE - dispatched by an a2a backend on this rank's own, and may leave its sum to a - mixer after it.""" - axis_sizes = token_axis_sizes() - attention = Layout.sharded_over( - TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes - ) +def _declaration(pattern: str, layer_idx: int): following = _stage_kind(pattern, layer_idx + 1) if is_attn_layer(pattern[layer_idx]): - owes = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1 - return StageDecl( - StageInput(attention, read=NORM_QUANT_READ), - StageOutput( - attention, - group=SumGroup.ATTN_TP if owes else None, - always_leaves=owes and following is StageKind.FFN, - leaves_for_next_layer=owes and following is StageKind.ATTENTION, - ), + return declare_attn( + mixer_exit=True, + next_kind=following, + ordinary_only=True, ) - sparse = pattern[layer_idx] == MOE - if sparse and not get_moe_a2a_backend().is_none(): - local = Layout.sharded_over( - TokenAxis.ATTN_DP, - TokenAxis.ATTN_CP, - TokenAxis.ATTN_TP_SCATTER, - axis_sizes=axis_sizes, - ) - return StageDecl(StageInput(local), StageOutput(local)) - full = Layout.sharded_over(axis_sizes=axis_sizes) - return StageDecl( - StageInput(full), - StageOutput( - full, - group=SumGroup.MOE_OUTPUT if sparse else SumGroup.TP, - leaves_for_next_layer=following is StageKind.ATTENTION, - leaves_for_reduce_scatter=True, - leaves_for_reduce_scatterv=True, - ), - ) - - -def layer_stage(pattern: str, layer_idx: int) -> LayerStage: - """The layer's stage and its two boundaries, from its own declaration and - the previous stage's.""" - rows = Layout.sharded_over( - TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=token_axis_sizes() - ) - return LayerStage( - kind=_stage_kind(pattern, layer_idx), - edges=stage_edges( - previous=( - _stage_decl(pattern, layer_idx - 1).output if layer_idx > 0 else None - ), - stage=_stage_decl(pattern, layer_idx), - rows=rows, - ), - enters_stack=layer_idx == 0, + return declare_ffn( + sparse=pattern[layer_idx] == MOE, + ordinary_only=True, + return_to_attention=True, ) def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int): - """The communicator of a layer that is one stage: only its own norm, and - boundaries built from the stages next to it in the pattern.""" - stage = layer_stage(pattern, layer_idx) - for_attn = stage.kind is StageKind.ATTENTION - boundaries = LayerCommunicator( - layer_facts=LayerFacts( - is_layer_sparse=pattern[layer_idx] == MOE, - is_last_layer=layer_idx == len(pattern) - 1, - ), - input_layernorm=layer_norm if for_attn else None, - post_attention_layernorm=None if for_attn else layer_norm, - # With attention TP > 1, the default gather adds the residual to one - # rank's partial in bf16 before the cross-rank sum. - force_layernorm_before_dp_gather=True, - allow_reduce_scatter=not for_attn, - stage=stage, - ) + from sglang.srt.layers import layernorm_sp + from sglang.srt.runtime_context import get_parallel - return boundaries.attn if for_attn else boundaries.ffn + if get_parallel().attn_cp_size > 1 or layernorm_sp.layernorm_sp_enabled(): + raise NotImplementedError("a Nemotron stage with attention CP or LayerNorm SP") + previous = _declaration(pattern, layer_idx - 1) if layer_idx > 0 else None + (boundary,) = make_stages( + ( + _declaration(pattern, layer_idx), + layer_norm, + {"force_layernorm_before_dp_gather": True}, + ), + previous=previous, + terminal=layer_idx == len(pattern) - 1, + ) + return boundary diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index afe1a56cac..0177eb237b 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -31,7 +31,7 @@ from sglang.kernels.ops.elementwise.elementwise import ( fused_gate_sigmoid_mul, fused_gate_sigmoid_mul_add, ) -from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo +from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages from sglang.srt.distributed import ( get_pp_indices, ) @@ -45,8 +45,9 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateList, ) from sglang.srt.layers.communicator import ( - LayerCommunicator, - LayerFacts, + declare_attn, + declare_ffn, + make_stages, ) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( @@ -1032,14 +1033,6 @@ class Qwen2MoeDecoderLayer(nn.Module): is_previous_layer_sparse = True is_next_layer_sparse = True - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - if self.is_layer_sparse: self.mlp = Qwen2MoeSparseMoeBlock( layer_id=layer_id, @@ -1060,12 +1053,21 @@ class Qwen2MoeDecoderLayer(nn.Module): self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, - allow_deferred_ffn_reduction=False, + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) def forward( @@ -1073,19 +1075,15 @@ class Qwen2MoeDecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> torch.Tensor: - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - captured_last_layer_outputs=captured_last_layer_outputs, - **kwargs, - ) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + captured_last_layer_outputs=captured_last_layer_outputs, + **kwargs, ) if hidden_states.shape[0] != 0: @@ -1095,15 +1093,14 @@ class Qwen2MoeDecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + with self.ffn_stage.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) + hidden_states = ffn_exit.finish(hidden_states) - return hidden_states, residual + return hidden_states class Qwen2MoeModel(nn.Module): @@ -1180,22 +1177,21 @@ class Qwen2MoeModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) aux_hidden_states = AuxHiddenStateList() if forward_batch.can_run_tbo: - hidden_states, residual = model_forward_maybe_tbo( + hidden_states = model_forward_stages( layers=self.layers, enable_tbo=True, positions=positions, forward_batch=forward_batch, hidden_states=hidden_states, - residual=residual, ) else: for i in range(self.start_layer, self.end_layer): @@ -1206,38 +1202,23 @@ class Qwen2MoeModel(nn.Module): ) with ctx: layer = self.layers[i] - hidden_states, residual = layer( + hidden_states = layer( positions, hidden_states, forward_batch, - residual, - captured_last_layer_outputs=( - aux_hidden_states - if getattr(layer, "_is_layer_to_capture", False) - else None - ), + captured_last_layer_outputs=aux_hidden_states + if getattr(layer, "_is_layer_to_capture", False) + else None, ) - last_layer = self.layers[self.end_layer - 1] - hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, - residual, - forward_batch, - preserve_declared=not self.pp_group.is_last_rank, - ) if not self.pp_group.is_last_rank: - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) + return residual_batch.to_pp(hidden_states, forward_batch) else: + hidden_states = residual_batch.finish(hidden_states, forward_batch) if hidden_states.shape[0] != 0: - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.norm + ) if len(aux_hidden_states) == 0: return hidden_states @@ -1314,7 +1295,7 @@ class Qwen2MoeForCausalLM(nn.Module): start, end = split_interval # embed if start == 0: - forward_batch.residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) if input_embeds is None: forward_batch.hidden_states = self.model.embed_tokens(input_ids) else: @@ -1324,22 +1305,17 @@ class Qwen2MoeForCausalLM(nn.Module): for i in range(start, end): with get_global_expert_distribution_recorder().with_current_layer(i): layer = self.model.layers[i] - forward_batch.hidden_states, forward_batch.residual = layer( - positions, - forward_batch.hidden_states, - forward_batch, - forward_batch.residual, + forward_batch.hidden_states = layer( + positions, forward_batch.hidden_states, forward_batch ) if end == self.model.config.num_hidden_layers: - forward_batch.hidden_states, forward_batch.residual = self.model.layers[ - end - 1 - ].layer_communicator.finish_layer_stack( - forward_batch.hidden_states, forward_batch.residual, forward_batch + forward_batch.hidden_states = residual_batch.finish( + forward_batch.hidden_states, forward_batch ) # norm - hidden_states, _ = self.model.norm( - forward_batch.hidden_states, forward_batch.residual + hidden_states = residual_batch.norm( + forward_batch.hidden_states, forward_batch, self.model.norm ) forward_batch.residual = None forward_batch.hidden_states = hidden_states diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 0fb9626600..6b808c90d7 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -40,18 +40,17 @@ from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader - -# Layers - Attention from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, AuxHiddenStateList, ) from sglang.srt.layers.communicator import ( - LayerCommunicator, - LayerFacts, + declare_attn, + declare_ffn, + make_stages, ) -# Configs +# Layers - Attention from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -271,7 +270,7 @@ def _enable_qwen35_fused_ar_quant() -> bool: (AR → RMSNorm → per-group quant) with either a single fused kernel (when the fully-fused variant is eligible) or a 2-kernel path (fused AR+RMSNorm + separate per-group quant) that still saves one - kernel launch vs. baseline. The LayerCommunicator gracefully falls back + kernel launch vs. baseline. The stage boundary gracefully falls back to ``forward_with_allreduce_fusion`` (plain AR+RMSNorm) when the fused quant helper returns ``None``, so turning this on never regresses the AR+RMSNorm fusion itself. @@ -1109,14 +1108,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module): else: raise ValueError(f"Invalid model type: {config.model_type}") - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = GemmaRMSNorm( config.hidden_size, eps=config.rms_norm_eps @@ -1128,33 +1119,44 @@ class Qwen3_5LinearDecoderLayer(nn.Module): _enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz) ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, - enable_fused_ar_quant=enable_fused_ar_quant, - fused_ar_quant_keep_bf16=enable_fused_ar_quant, - fusions=_layer_fusions(config, is_nextn), + boundary_fusions = _layer_fusions(config, is_nextn) + self.attn_stage, self.ffn_stage = make_stages( + ( + declare_attn(), + self.input_layernorm, + { + "enable_fused_ar_quant": enable_fused_ar_quant, + "fused_ar_quant_keep_bf16": enable_fused_ar_quant, + "fusions": boundary_fusions, + }, + ), + ( + declare_ffn( + sparse=is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + {"fusions": boundary_fusions}, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) def forward( self, hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], **kwargs, ): forward_batch = kwargs.get("forward_batch", None) - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - captured_last_layer_outputs=kwargs.get( - "captured_last_layer_outputs", None - ), - ) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + captured_last_layer_outputs=kwargs.get("captured_last_layer_outputs", None), ) # fused AR+quant hands down a (fp8, scale) / (bf16, fp8, scale) tuple @@ -1166,11 +1168,10 @@ class Qwen3_5LinearDecoderLayer(nn.Module): ) # Fully Connected - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + with self.ffn_stage.exit(forward_batch) as ffn_exit: if isinstance(self.mlp, Qwen2MoeSparseMoeBlock): hidden_states = self.mlp( hidden_states, @@ -1179,7 +1180,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module): ) else: hidden_states = self.mlp(hidden_states) - return ffn_exit.finish(hidden_states, residual) + return ffn_exit.finish(hidden_states) class Qwen3_5AttentionDecoderLayer(nn.Module): @@ -1311,14 +1312,6 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): else: raise ValueError(f"Invalid model type: {config.model_type}") - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = GemmaRMSNorm( config.hidden_size, eps=config.rms_norm_eps @@ -1332,14 +1325,31 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): enable_fused_ar_quant = ( _enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj) ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, - enable_fused_ar_quant=enable_fused_ar_quant, - fused_ar_quant_keep_bf16=False, - fusions=_layer_fusions(config, is_nextn), + boundary_fusions = _layer_fusions(config, is_nextn) + self.attn_stage, self.ffn_stage = make_stages( + ( + declare_attn(), + self.input_layernorm, + { + "enable_fused_ar_quant": enable_fused_ar_quant, + "fused_ar_quant_keep_bf16": False, + "fusions": boundary_fusions, + }, + ), + ( + declare_ffn( + sparse=is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + {"fusions": boundary_fusions}, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) self.alt_stream = alt_stream @@ -1543,18 +1553,14 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): self, positions: torch.Tensor, hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], forward_batch: ForwardBatch, captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, **kwargs, ): - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - captured_last_layer_outputs=captured_last_layer_outputs, - ) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + captured_last_layer_outputs=captured_last_layer_outputs, ) # fused AR+quant hands down a (fp8, scale) / (bf16, fp8, scale) tuple @@ -1567,10 +1573,9 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): ) # Fully Connected - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + with self.ffn_stage.exit(forward_batch) as ffn_exit: if isinstance(self.mlp, Qwen2MoeSparseMoeBlock): hidden_states = self.mlp( hidden_states, @@ -1579,7 +1584,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): ) else: hidden_states = self.mlp(hidden_states) - return ffn_exit.finish(hidden_states, residual) + return ffn_exit.finish(hidden_states) ALL_DECODER_LAYER_TYPES = { @@ -1747,6 +1752,7 @@ class Qwen3_5ForCausalLM(nn.Module): # Every layer was checked above. can_defer_finalize=lambda layer: True, label="Qwen3.5", + terminal_finalize=True, ) # Final normalization @@ -1821,12 +1827,12 @@ class Qwen3_5ForCausalLM(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) aux_hidden_states = AuxHiddenStateList() # Pass through decoder layers @@ -1835,16 +1841,13 @@ class Qwen3_5ForCausalLM(nn.Module): with get_global_expert_distribution_recorder().with_current_layer( layer_idx ): - hidden_states, residual = layer( + hidden_states = layer( positions=positions, hidden_states=hidden_states, - residual=residual, forward_batch=forward_batch, - captured_last_layer_outputs=( - aux_hidden_states - if getattr(layer, "_is_layer_to_capture", False) - else None - ), + captured_last_layer_outputs=aux_hidden_states + if getattr(layer, "_is_layer_to_capture", False) + else None, ) # Process deepstack embeddings if provided @@ -1854,52 +1857,22 @@ class Qwen3_5ForCausalLM(nn.Module): and layer_idx < 3 ): sep = self.hidden_size * layer_idx - hidden_states, residual = layer.layer_communicator.add_to_output( + hidden_states = residual_batch.add_to_output( hidden_states, - residual, + forward_batch, input_deepstack_embeds[:, sep : sep + self.hidden_size], ) - last_layer = self.layers[self.end_layer - 1] - hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, - residual, - forward_batch, - # The final norm below finalizes a deferred MoE output in its kernel. - final_norm_takes_handoff=self.flashinfer_mnnvl_cutedsl_fusion is not None - and self.pp_group.is_last_rank, - preserve_declared=not self.pp_group.is_last_rank, - ) - - # Return intermediate tensors for pipeline parallelism if not self.pp_group.is_last_rank: - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) + return residual_batch.to_pp(hidden_states, forward_batch) - # The final layer has no successor to consume its deferred MoE tail. - is_deferred_finalize = False - if self.flashinfer_mnnvl_cutedsl_fusion is not None: - from sglang.srt.layers.communicator.fusions.cutedsl import ( - MoeFinalizeHandoff, - ) - - is_deferred_finalize = isinstance(hidden_states, MoeFinalizeHandoff) - - if is_deferred_finalize: - if residual is None: - raise RuntimeError("invalid final deferred MoE handoff") - hidden_states, _ = self.flashinfer_mnnvl_cutedsl_fusion.finalize( - handoff=hidden_states, residual=residual, gamma=self.norm.gemma_weight - ) - elif hidden_states.shape[0] != 0: - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = residual_batch.norm( + hidden_states, + forward_batch, + self.norm, + handoff_norm=self.flashinfer_mnnvl_cutedsl_fusion, + skip_empty=True, + ) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index f3ee5b1e74..47eea51b15 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -30,7 +30,11 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator -from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator import ( + declare_attn, + declare_ffn, + make_stages, +) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( @@ -842,14 +846,6 @@ class Qwen3MoeDecoderLayer(nn.Module): is_previous_layer_sparse = True is_next_layer_sparse = True - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - if self.is_layer_sparse: self.mlp = Qwen3MoeSparseMoeBlock( layer_id=self.layer_id, @@ -870,11 +866,21 @@ class Qwen3MoeDecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) def forward( @@ -882,21 +888,17 @@ class Qwen3MoeDecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, capture_output=None, **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> torch.Tensor: - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - captured_last_layer_outputs=captured_last_layer_outputs, - capture_output=capture_output, - **kwargs, - ) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + captured_last_layer_outputs=captured_last_layer_outputs, + capture_output=capture_output, + **kwargs, ) if hidden_states.shape[0] != 0: @@ -906,15 +908,14 @@ class Qwen3MoeDecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + with self.ffn_stage.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) + hidden_states = ffn_exit.finish(hidden_states) - return hidden_states, residual + return hidden_states def op_comm_prepare_attn( self, @@ -922,11 +923,10 @@ class Qwen3MoeDecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], tbo_subbatch_index: Optional[int] = None, ): - state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = ( - self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch) + state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare( + hidden_states, forward_batch ) state.update( dict( @@ -937,25 +937,21 @@ class Qwen3MoeDecoderLayer(nn.Module): ) def op_comm_prepare_mlp(self, state): - state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = ( - self.layer_communicator.prepare_mlp( - state.pop("hidden_states_after_attn"), - state.pop("residual_after_input_ln"), - state.forward_batch, - ) + hidden_states = self.attn_stage.finish( + state.pop("hidden_states_after_attn"), state.forward_batch + ) + state.hidden_states_mlp_input = self.ffn_stage.prepare( + hidden_states, state.forward_batch ) def op_comm_postprocess_layer(self, state): - hidden_states, residual = self.layer_communicator.postprocess_layer( - state.pop("hidden_states_mlp_output"), - state.pop("residual_after_comm_pre_mlp"), - state.forward_batch, + hidden_states = self.ffn_stage.postprocess( + state.pop("hidden_states_mlp_output"), state.forward_batch ) output = dict( positions=state.positions, hidden_states=hidden_states, - residual=residual, forward_batch=state.forward_batch, tbo_subbatch_index=state.tbo_subbatch_index, ) @@ -1088,7 +1084,7 @@ class Qwen3MoeForCausalLM(nn.Module): start, end = split_interval # embed if start == 0: - forward_batch.residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) if input_embeds is None: forward_batch.hidden_states = self.model.embed_tokens(input_ids) else: @@ -1098,22 +1094,17 @@ class Qwen3MoeForCausalLM(nn.Module): for i in range(start, end): with get_global_expert_distribution_recorder().with_current_layer(i): layer = self.model.layers[i] - forward_batch.hidden_states, forward_batch.residual = layer( - positions, - forward_batch.hidden_states, - forward_batch, - forward_batch.residual, + forward_batch.hidden_states = layer( + positions, forward_batch.hidden_states, forward_batch ) if end == self.model.config.num_hidden_layers: - forward_batch.hidden_states, forward_batch.residual = self.model.layers[ - end - 1 - ].layer_communicator.finish_layer_stack( - forward_batch.hidden_states, forward_batch.residual, forward_batch + forward_batch.hidden_states = residual_batch.finish( + forward_batch.hidden_states, forward_batch ) # norm - hidden_states, _ = self.model.norm( - forward_batch.hidden_states, forward_batch.residual + hidden_states = residual_batch.norm( + forward_batch.hidden_states, forward_batch, self.model.norm ) forward_batch.residual = None forward_batch.hidden_states = hidden_states diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index 73f16b58ce..507dfdfa8e 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -19,7 +19,11 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, AuxHiddenStateList, ) -from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator import ( + declare_attn, + declare_ffn, + make_stages, +) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -471,13 +475,11 @@ class Qwen3GatedDeltaNet(nn.Module): def _apply_qwen3_next_mlp( layer: nn.Module, hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], forward_batch: ForwardBatch, -) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - hidden_states, residual = layer.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) - with layer.layer_communicator.ffn_exit(forward_batch) as ffn_exit: +) -> torch.Tensor: + hidden_states = layer.attn_stage.finish(hidden_states, forward_batch) + hidden_states = layer.ffn_stage.prepare(hidden_states, forward_batch) + with layer.ffn_stage.exit(forward_batch) as ffn_exit: if isinstance(layer.mlp, Qwen2MoeSparseMoeBlock): hidden_states = layer.mlp( hidden_states, @@ -485,9 +487,9 @@ def _apply_qwen3_next_mlp( ) else: hidden_states = layer.mlp(hidden_states) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) + hidden_states = ffn_exit.finish(hidden_states) - return hidden_states, residual + return hidden_states class Qwen3HybridLinearDecoderLayer(nn.Module): @@ -512,14 +514,6 @@ class Qwen3HybridLinearDecoderLayer(nn.Module): is_next_layer_sparse = True self.layer_id = layer_id - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - if self.is_layer_sparse: self.mlp = Qwen2MoeSparseMoeBlock( layer_id=layer_id, @@ -543,29 +537,35 @@ class Qwen3HybridLinearDecoderLayer(nn.Module): self.post_attention_layernorm = GemmaRMSNorm( config.hidden_size, eps=config.rms_norm_eps ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) def forward( self, hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, **kwargs, ): forward_batch = kwargs.get("forward_batch", None) - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - captured_last_layer_outputs=captured_last_layer_outputs, - ) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + captured_last_layer_outputs=captured_last_layer_outputs, ) if not forward_batch.forward_mode.is_idle(): @@ -573,11 +573,9 @@ class Qwen3HybridLinearDecoderLayer(nn.Module): hidden_states, forward_batch, ) - hidden_states, residual = _apply_qwen3_next_mlp( - self, hidden_states, residual, forward_batch - ) + hidden_states = _apply_qwen3_next_mlp(self, hidden_states, forward_batch) - return hidden_states, residual + return hidden_states class Qwen3HybridAttentionDecoderLayer(nn.Module): @@ -680,14 +678,6 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): is_previous_layer_sparse = True is_next_layer_sparse = True - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - if self.is_layer_sparse: self.mlp = Qwen2MoeSparseMoeBlock( layer_id=layer_id, @@ -715,11 +705,21 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) self.alt_stream = alt_stream @@ -828,18 +828,14 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): self, positions: torch.Tensor, hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], forward_batch: ForwardBatch, captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, **kwargs: Any, ): - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - captured_last_layer_outputs=captured_last_layer_outputs, - ) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + captured_last_layer_outputs=captured_last_layer_outputs, ) if not forward_batch.forward_mode.is_idle(): @@ -849,11 +845,9 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states, residual = _apply_qwen3_next_mlp( - self, hidden_states, residual, forward_batch - ) + hidden_states = _apply_qwen3_next_mlp(self, hidden_states, forward_batch) - return hidden_states, residual + return hidden_states ALL_DECODER_LAYER_TYPES = { @@ -934,33 +928,24 @@ class Qwen3NextModel(nn.Module): else: hidden_states = self.embed_tokens(input_ids) - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) aux_hidden_states = AuxHiddenStateList() for i in range(len(self.layers)): layer = self.layers[i] with get_global_expert_distribution_recorder().with_current_layer(i): - hidden_states, residual = layer( + hidden_states = layer( layer_id=i, positions=positions, hidden_states=hidden_states, - residual=residual, forward_batch=forward_batch, - captured_last_layer_outputs=( - aux_hidden_states - if getattr(layer, "_is_layer_to_capture", False) - else None - ), + captured_last_layer_outputs=aux_hidden_states + if getattr(layer, "_is_layer_to_capture", False) + else None, ) - last_layer = self.layers[-1] - hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch - ) + hidden_states = residual_batch.finish(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/qwen3_vl_moe.py b/python/sglang/srt/models/qwen3_vl_moe.py index da7a93a4ca..1226b1684c 100644 --- a/python/sglang/srt/models/qwen3_vl_moe.py +++ b/python/sglang/srt/models/qwen3_vl_moe.py @@ -96,12 +96,12 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) aux_hidden_states = AuxHiddenStateList() for layer_idx, layer in enumerate( @@ -116,11 +116,10 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): deepstack_embeds = self.get_deepstack_embeds( layer_idx - 1, input_deepstack_embeds ) - hidden_states, residual = layer( + hidden_states = layer( positions, hidden_states, forward_batch, - residual, post_residual_addition=deepstack_embeds, capture_output=aux_hidden_states.capture if layer_idx in self.layers_to_capture @@ -129,11 +128,10 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): else: # Inference path: add deepstack directly to hidden_states at the end of the layer # (original, grounding-correct order). - hidden_states, residual = layer( + hidden_states = layer( positions, hidden_states, forward_batch, - residual, capture_output=aux_hidden_states.capture if layer_idx in self.layers_to_capture else None, @@ -143,9 +141,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): and layer_idx in self.deepstack_embed_to_decoder_layer ): sep = self.hidden_size * layer_idx - hidden_states, residual = layer.layer_communicator.add_to_output( + hidden_states = residual_batch.add_to_output( hidden_states, - residual, + forward_batch, input_deepstack_embeds[:, sep : sep + self.hidden_size], ) @@ -156,28 +154,17 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): else None ) - last_layer = self.layers[self.end_layer - 1] - hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, - residual, - forward_batch, - preserve_declared=not self.pp_group.is_last_rank, - ) if not self.pp_group.is_last_rank: - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) + return residual_batch.to_pp(hidden_states, forward_batch) else: + hidden_states = residual_batch.finish(hidden_states, forward_batch) if hidden_states.shape[0] != 0: - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm( - hidden_states, residual, post_residual_addition=last_deepstack - ) + hidden_states = residual_batch.norm( + hidden_states, + forward_batch, + self.norm, + post_residual_addition=last_deepstack, + ) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py index 9779b6e62a..69397d5963 100644 --- a/python/sglang/srt/models/qwen4_exp.py +++ b/python/sglang/srt/models/qwen4_exp.py @@ -1320,7 +1320,8 @@ class Qwen4ExpLayerExtensionMixin: for attr_name in ( "input_layernorm", "post_attention_layernorm", - "layer_communicator", + "attn_stage", + "ffn_stage", ): if hasattr(self, attr_name): delattr(self, attr_name) diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index beccb75323..60c136ca51 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -20,9 +20,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( - LayerCommunicator, - LayerFacts, + declare_attn, + declare_ffn, enable_moe_dense_fully_dp, + make_stages, ) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( @@ -1048,19 +1049,26 @@ class SarvamMoEMLADecoderLayer(nn.Module): ) self.attn_tp_size = get_parallel().attn_tp_size - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - qkv_latent_func=self.self_attn.prepare_qkv_latent, - allow_reduce_scatter=True, + + self.attn_stage, self.ffn_stage = make_stages( + ( + declare_attn(), + self.input_layernorm, + {"qkv_latent_func": self.self_attn.prepare_qkv_latent}, + ), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) def forward( @@ -1068,21 +1076,17 @@ class SarvamMoEMLADecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor]: - hidden_states, residual = self.layer_communicator.prepare_attn( - hidden_states, residual, forward_batch - ) + ) -> torch.Tensor: + hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( positions=positions, hidden_states=hidden_states, forward_batch=forward_batch, ) - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + with self.ffn_stage.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) if ( not self.is_layer_sparse @@ -1091,8 +1095,8 @@ class SarvamMoEMLADecoderLayer(nn.Module): and not ffn_exit.fuse_mlp_allreduce ): hidden_states = tensor_model_parallel_all_reduce(hidden_states) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) - return hidden_states, residual + hidden_states = ffn_exit.finish(hidden_states) + return hidden_states class SarvamMLAModel(nn.Module): @@ -1152,36 +1156,23 @@ class SarvamMLAModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) for i in range(self.start_layer, self.end_layer): layer = self.layers[i] - hidden_states, residual = layer( - positions, hidden_states, forward_batch, residual - ) + hidden_states = layer(positions, hidden_states, forward_batch) - last_layer = self.layers[self.end_layer - 1] - hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, - residual, - forward_batch, - preserve_declared=not self.pp_group.is_last_rank, - ) if not self.pp_group.is_last_rank: - return PPProxyTensors( - {"hidden_states": hidden_states, "residual": residual} - ) + return residual_batch.to_pp(hidden_states, forward_batch) + hidden_states = residual_batch.finish(hidden_states, forward_batch) if hidden_states.shape[0] != 0: - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) return hidden_states @@ -1267,36 +1258,26 @@ class SarvamMLAForCausalLM(nn.Module): ) -> Optional[LogitsProcessorOutput]: start, end = split_interval if start == 0: + residual_batch.start(forward_batch) if input_embeds is None: forward_batch.hidden_states = self.model.embed_tokens(input_ids) else: forward_batch.hidden_states = input_embeds - forward_batch.residual = residual_batch.start(forward_batch) for i in range(start, end): with get_global_expert_distribution_recorder().with_current_layer(i): layer = self.model.layers[i] - forward_batch.hidden_states, forward_batch.residual = layer( - positions, - forward_batch.hidden_states, - forward_batch, - forward_batch.residual, + forward_batch.hidden_states = layer( + positions, forward_batch.hidden_states, forward_batch ) if end == self.model.config.num_hidden_layers: - forward_batch.hidden_states, forward_batch.residual = self.model.layers[ - end - 1 - ].layer_communicator.finish_layer_stack( - forward_batch.hidden_states, forward_batch.residual, forward_batch + forward_batch.hidden_states = residual_batch.finish( + forward_batch.hidden_states, forward_batch + ) + forward_batch.hidden_states = residual_batch.norm( + forward_batch.hidden_states, forward_batch, self.model.norm ) - if forward_batch.residual is None: - hidden_states = self.model.norm(forward_batch.hidden_states) - else: - hidden_states, _ = self.model.norm( - forward_batch.hidden_states, forward_batch.residual - ) - forward_batch.residual = None - forward_batch.hidden_states = hidden_states return self.logits_processor( input_ids, forward_batch.hidden_states, self.lm_head, forward_batch ) diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 4c53cba411..6915dc6853 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -11,7 +11,11 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator import ( + declare_attn, + declare_ffn, + make_stages, +) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -561,21 +565,23 @@ class Step3p5DecoderLayer(nn.Module): ) # An MTP draft is a one-layer model; layer_id still indexes its config. - self.layer_facts = LayerFacts.init_new( - layer_id=0 if is_nextn else layer_id, - num_layers=1 if is_nextn else config.num_hidden_layers, - is_layer_sparse=self.is_moe_layer, - is_previous_layer_sparse=self.is_previous_layer_sparse, - is_next_layer_sparse=self.is_next_layer_sparse, - ) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, - # The dense MLP all-reduces its own output unless postprocess - # reduce-scatters it; it never leaves the sum to the next layer. - allow_deferred_ffn_reduction=self.use_moe, + + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn( + sparse=self.is_moe_layer, + next_sparse=self.is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=self.is_previous_layer_sparse, next_sparse=self.is_moe_layer + ) + if not (0 if is_nextn else layer_id) == 0 + else None, + terminal=(0 if is_nextn else layer_id) + == (1 if is_nextn else config.num_hidden_layers) - 1, ) self.layer_id = layer_id @@ -585,15 +591,11 @@ class Step3p5DecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], post_residual_addition: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> torch.Tensor: # Self Attention - hidden_states, residual = self.layer_communicator.prepare_attn( - hidden_states, - residual, - forward_batch, - post_residual_addition=post_residual_addition, + hidden_states = self.attn_stage.prepare( + hidden_states, forward_batch, post_residual_addition=post_residual_addition ) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( @@ -602,14 +604,11 @@ class Step3p5DecoderLayer(nn.Module): forward_batch=forward_batch, ) # Fully Connected - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, - residual, - forward_batch, - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) if self.use_moe: - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + with self.ffn_stage.exit(forward_batch) as ffn_exit: # Both share_expert and MoE return unreduced (TP-partial) outputs. # Combine them first, then do a single all-reduce — saving one # full-TP all-reduce per layer. @@ -620,11 +619,11 @@ class Step3p5DecoderLayer(nn.Module): hidden_states = moe_output + share_output if not ffn_exit.fuse_mlp_allreduce and not ffn_exit.mlp_reduce_scatter: hidden_states = tensor_model_parallel_all_reduce(hidden_states) - return ffn_exit.finish(hidden_states, residual) + return ffn_exit.finish(hidden_states) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + with self.ffn_stage.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states) - return ffn_exit.finish(hidden_states, residual) + return ffn_exit.finish(hidden_states) class Step3p5Model(nn.Module): @@ -698,55 +697,33 @@ class Step3p5Model(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) for i in range(self.start_layer, self.end_layer): layer = self.layers[i] - hidden_states, residual = layer( - positions, - hidden_states, - forward_batch, - residual, - ) + hidden_states = layer(positions, hidden_states, forward_batch) # break - last_layer = self.layers[self.end_layer - 1] - hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, - residual, - forward_batch, - preserve_declared=not self.pp_group.is_last_rank, - ) if not self.pp_group.is_last_rank: - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) + return residual_batch.to_pp(hidden_states, forward_batch) else: + hidden_states = residual_batch.finish(hidden_states, forward_batch) hidden_states_before_norm = None if not self.pp_group.is_last_rank: - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) + return residual_batch.to_pp(hidden_states, forward_batch) else: if hidden_states.shape[0] > 0: # if forward_batch.return_hidden_states_before_norm: - hidden_states_before_norm = last_layer.layer_communicator.snapshot( - hidden_states, residual + hidden_states_before_norm = residual_batch.snapshot( + hidden_states, forward_batch + ) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.norm ) - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) return hidden_states, hidden_states_before_norm diff --git a/python/sglang/srt/models/step3p5_mtp.py b/python/sglang/srt/models/step3p5_mtp.py index 07356a85c1..08fd786fbe 100644 --- a/python/sglang/srt/models/step3p5_mtp.py +++ b/python/sglang/srt/models/step3p5_mtp.py @@ -110,25 +110,22 @@ class Step3p5AMultiTokenPredictor(nn.Module): dim=-1, ) ) - hidden_states, residual = self.mtp_block( + residual_batch.start(forward_batch) + hidden_states = self.mtp_block( positions=positions, hidden_states=hidden_states, forward_batch=forward_batch, - residual=residual_batch.start(forward_batch), - ) - hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch ) + hidden_states = residual_batch.finish(hidden_states, forward_batch) hidden_states_before_norm = None if not forward_batch.forward_mode.is_idle(): # if forward_batch.return_hidden_states_before_norm: - hidden_states_before_norm = self.mtp_block.layer_communicator.snapshot( - hidden_states, residual + hidden_states_before_norm = residual_batch.snapshot( + hidden_states, forward_batch + ) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.shared_head.norm ) - if residual is not None: - hidden_states, _ = self.shared_head.norm(hidden_states, residual) - else: - hidden_states = self.shared_head.norm(hidden_states) return hidden_states, hidden_states_before_norm diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index d956299d06..7de43944bc 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -40,9 +40,10 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( - LayerCommunicator, - LayerFacts, + declare_attn, + declare_ffn, enable_moe_dense_fully_dp, + make_stages, ) from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled @@ -860,11 +861,10 @@ class XllmMLP(nn.Module): def forward( self, x, - use_reduce_scatter: bool = False, ): gate_up, _ = self.gate_up_proj(x) x = self.act_fn(gate_up) - x, _ = self.down_proj(x, skip_all_reduce=use_reduce_scatter) + x, _ = self.down_proj(x) return x @@ -1041,7 +1041,6 @@ class XllmSparseMoeBlock(nn.Module): self, hidden_states: torch.Tensor, forward_batch: Optional[ForwardBatch] = None, - use_reduce_scatter: bool = False, ) -> torch.Tensor: num_tokens, hidden_dim = hidden_states.shape hidden_states = hidden_states.view(-1, hidden_dim) @@ -1061,8 +1060,7 @@ class XllmSparseMoeBlock(nn.Module): if shared_output is not None: final_hidden_states += shared_output - if not use_reduce_scatter: - final_hidden_states = reduce_moe_output(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_dim) @@ -1506,14 +1504,6 @@ class XllmDecoderLayer(nn.Module): is_previous_layer_sparse = _is_sparse(layer_id - 1) is_next_layer_sparse = _is_sparse(layer_id + 1) - self.layer_facts = LayerFacts.init_new( - layer_id=layer_id, - num_layers=config.num_hidden_layers, - is_layer_sparse=self.is_layer_sparse, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - if self.is_layer_sparse: self.mlp = XllmSparseMoeBlock( layer_id=layer_id, @@ -1538,12 +1528,21 @@ class XllmDecoderLayer(nn.Module): self.input_layernorm = _make_norm(config) self.post_attention_layernorm = _make_norm(config) - self.layer_communicator = LayerCommunicator( - layer_facts=self.layer_facts, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, - allow_reduce_scatter=True, - allow_deferred_ffn_reduction=False, + self.attn_stage, self.ffn_stage = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn( + sparse=self.is_layer_sparse, + next_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) def forward( @@ -1551,13 +1550,8 @@ class XllmDecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor]: - hidden_states, residual = self.layer_communicator.prepare_attn( - hidden_states, - residual, - forward_batch, - ) + ) -> torch.Tensor: + hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( @@ -1566,23 +1560,17 @@ class XllmDecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + hidden_states = self.attn_stage.finish(hidden_states, forward_batch) + hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: - use_reduce_scatter = ffn_exit.mlp_reduce_scatter + with self.ffn_stage.exit(forward_batch) as ffn_exit: if isinstance(self.mlp, XllmMLP): - hidden_states = self.mlp( - hidden_states, use_reduce_scatter=use_reduce_scatter - ) + hidden_states = self.mlp(hidden_states) else: - hidden_states = self.mlp( - hidden_states, forward_batch, use_reduce_scatter - ) - hidden_states, residual = ffn_exit.finish(hidden_states, residual) + hidden_states = self.mlp(hidden_states, forward_batch) + hidden_states = ffn_exit.finish(hidden_states) - return hidden_states, residual + return hidden_states class XllmModel(nn.Module): @@ -1638,12 +1626,12 @@ class XllmModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = residual_batch.start(forward_batch) + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = self.layers[ - self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + pp_proxy_tensors, forward_batch + ) for i in range(self.start_layer, self.end_layer): ctx = ( @@ -1653,34 +1641,16 @@ class XllmModel(nn.Module): ) with ctx: layer = self.layers[i] - hidden_states, residual = layer( - positions, - hidden_states, - forward_batch, - residual, - ) + hidden_states = layer(positions, hidden_states, forward_batch) - hidden_states, residual = self.layers[ - self.end_layer - 1 - ].layer_communicator.finish_layer_stack( - hidden_states, - residual, - forward_batch, - preserve_declared=not self.pp_group.is_last_rank, - ) if not self.pp_group.is_last_rank: - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) + return residual_batch.to_pp(hidden_states, forward_batch) else: + hidden_states = residual_batch.finish(hidden_states, forward_batch) if hidden_states.shape[0] != 0: - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = residual_batch.norm( + hidden_states, forward_batch, self.norm + ) return hidden_states diff --git a/python/sglang/test/boundary_fixtures.py b/python/sglang/test/boundary_fixtures.py index 5ae4de8887..997a9834b6 100644 --- a/python/sglang/test/boundary_fixtures.py +++ b/python/sglang/test/boundary_fixtures.py @@ -71,6 +71,7 @@ def stub_plan(): plan._fusion_rows = None plan._cp_steps = plan._sp_steps = plan._input_scattered_steps = None plan.is_first_layer = False + plan.is_last_layer = False plan.residual_in_hidden = False # Keep production variant selection while tests replace individual paths. plan._batch_steps = lambda batch: { diff --git a/python/sglang/test/communicator_patch.py b/python/sglang/test/communicator_patch.py index 521e40406c..0286b1d58f 100644 --- a/python/sglang/test/communicator_patch.py +++ b/python/sglang/test/communicator_patch.py @@ -35,8 +35,6 @@ from sglang.srt.layers.communicator.residual import ( ) COMMUNICATOR_MODULES = ( - layer, - legacy_stage, communicator, allreduce, exit, @@ -54,6 +52,8 @@ COMMUNICATOR_MODULES = ( construction, factories, stage, + layer, + legacy_stage, batch, stream, ) diff --git a/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py b/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py index 80bce38c24..22c9cfa4ab 100644 --- a/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py +++ b/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py @@ -68,15 +68,12 @@ patches: - target: sglang.srt.models.qwen3_moe.Qwen3MoeDecoderLayer.forward edits: - match: | - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - captured_last_layer_outputs=captured_last_layer_outputs, - capture_output=capture_output, - **kwargs, - ) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + captured_last_layer_outputs=captured_last_layer_outputs, + capture_output=capture_output, + **kwargs, ) append: "dumper.dump('layer_input', hidden_states, dims='t h # tp:replicated')" - match: | @@ -86,10 +83,7 @@ patches: forward_batch=forward_batch, ) append: "dumper.dump('attn_output', hidden_states, dims='t h[attn_tp:partial] # tp:replicated')" - - match: | - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + - match: "hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)" append: "dumper.dump('pre_mlp_residual', hidden_states, dims='t h # tp:replicated')" - match: "hidden_states = self.mlp(hidden_states, forward_batch)" append: "dumper.dump('mlp_output', hidden_states, dims='t h[moe_tp:partial] # tp:replicated')" @@ -120,21 +114,18 @@ patches: # # Attn tensors are NOT TP-sharded (attn_tp_size=1). # mlp_output is still moe_tp:partial — the reduce-scatter happens in - # postprocess_layer(), after the dump point. - # layer_input is dumped after prepare_attn which DP-distributes tokens, + # the output boundary, after the dump point. + # layer_input is dumped after attention preparation, which DP-distributes tokens, # so it needs dp:=attn_dp to filter to the non-empty DP rank. - target: sglang.srt.models.qwen3_moe.Qwen3MoeDecoderLayer.forward edits: - match: | - hidden_states, residual = ( - self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( - hidden_states, - residual, - forward_batch, - captured_last_layer_outputs=captured_last_layer_outputs, - capture_output=capture_output, - **kwargs, - ) + hidden_states = self.attn_stage.prepare( + hidden_states, + forward_batch, + captured_last_layer_outputs=captured_last_layer_outputs, + capture_output=capture_output, + **kwargs, ) append: "dumper.dump('layer_input', hidden_states, dims='t h # tp:replicated dp:=attn_dp')" - match: | @@ -144,10 +135,7 @@ patches: forward_batch=forward_batch, ) append: "dumper.dump('attn_output', hidden_states, dims='t h # tp:replicated')" - - match: | - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) + - match: "hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)" append: "dumper.dump('pre_mlp_residual', hidden_states, dims='t h # tp:replicated')" - match: "hidden_states = self.mlp(hidden_states, forward_batch)" append: "dumper.dump('mlp_output', hidden_states, dims='t h[moe_tp:partial] # tp:replicated')" @@ -183,7 +171,7 @@ class TestSourcePatcherE2ESGLang: In dp-attention mode (attn_tp_size=1, attn_dp_size=2), attention tensors are NOT TP-sharded and mlp_output is still moe_tp:partial - (the reduce-scatter happens in postprocess_layer, after the dump + (the reduce-scatter happens at the output boundary, after the dump point). A separate patch config with corrected dims is used for the target. diff --git a/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py index f7e82d48da..a338254490 100644 --- a/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py +++ b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py @@ -24,7 +24,7 @@ class TestTboEntryReducesItsInput(CustomTestCase): residual = torch.ones(4, 3) stream = ResidualStream(residual) hidden = stream.leave(UnreducedOutput(torch.ones(4, 3), group=group), ADD) - parent_batch = SimpleNamespace(residual_stream=stream) + batch = SimpleNamespace(residual_stream=stream, global_forward_mode=None) parts_seen = [] def split(hidden_states, residual, **kwargs): @@ -40,12 +40,15 @@ class TestTboEntryReducesItsInput(CustomTestCase): ] def execute(inputs_arr, **kwargs): - self.assertIsNone(parent_batch.residual_stream) + self.assertIsNone(batch.residual_stream) parts_seen.extend(inputs_arr) - self.assertIsNot(inputs_arr[0]["residual"], inputs_arr[1]["residual"]) + self.assertIsNot( + inputs_arr[0]["forward_batch"].residual_stream, + inputs_arr[1]["forward_batch"].residual_stream, + ) for part in inputs_arr: - state = part["residual"] - self.assertIs(state, part["forward_batch"].residual_stream) + self.assertNotIn("residual", part) + state = part["forward_batch"].residual_stream value = part["hidden_states"] self.assertIsNot(state.pending, stream.pending) self.assertIs(state.pending.update, ADD) @@ -60,6 +63,13 @@ class TestTboEntryReducesItsInput(CustomTestCase): return inputs_arr with ( + patch.object( + tbo.OperationsStrategy, + "init_new_tbo", + return_value=SimpleNamespace( + deep_gemm_num_sms=None, operations=[], tbo_delta_stages=0 + ), + ), patch.object(tbo, "_model_forward_tbo_split_inputs", split), patch.object(tbo, "execute_overlapped_operations", execute), patch.object( @@ -68,21 +78,19 @@ class TestTboEntryReducesItsInput(CustomTestCase): lambda _: empty_context(), ), ): - merged, output = tbo._model_forward_tbo( - inputs=dict( - hidden_states=hidden, - residual=stream, - positions=None, - forward_batch=parent_batch, - zero_allocator=None, - ), - operations_strategy=SimpleNamespace( - deep_gemm_num_sms=None, operations=[], tbo_delta_stages=0 - ), - layer_input_rows=Layout(frozenset()), + merged = tbo.model_forward_stages( + layers=[ + SimpleNamespace( + attn_stage=SimpleNamespace(input_rows=Layout(frozenset())) + ) + ], + enable_tbo=True, + positions=None, + hidden_states=hidden, + forward_batch=batch, ) + output = batch.residual_stream self.assertEqual(len(parts_seen), 2) - self.assertIs(parent_batch.residual_stream, output) self.assertTrue( all(part["forward_batch"].residual_stream is None for part in parts_seen) ) diff --git a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py index caa1e4d72e..034e52acc8 100644 --- a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py @@ -9,13 +9,20 @@ import torch from sglang.srt.layers.communicator import ( ADD, NORM_QUANT_READ, - LayerCommunicator, Layout, + StageKind, SumGroup, TokenAxis, UnreducedOutput, reduce_output, ) +from sglang.srt.layers.communicator.construction import BatchVariant +from sglang.srt.layers.communicator.fusions.allreduce import ( + attention_fusions, + complete_attention_input, + complete_ffn_input, + ffn_fusions, +) from sglang.srt.layers.communicator.fusions.cutedsl import ( CuteDSLFusion, MoeFinalizeHandoff, @@ -24,6 +31,7 @@ from sglang.srt.layers.communicator.fusions.cutedsl import ( ) from sglang.srt.layers.communicator.ops import _consumer_step, _read_input from sglang.srt.layers.communicator.residual import batch as residual_batch +from sglang.srt.layers.communicator.residual.access import finish_layer_stack from sglang.srt.layers.flashinfer_mnnvl_cutedsl import ( FlashInferMNNVLCuteDSLARFusion, _retargeted_config, @@ -34,6 +42,7 @@ from sglang.srt.model_executor.cuda_graph_config import CudaGraphConfig, PhaseCo from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import get_forward, publish, reset_context from sglang.srt.server_args import ServerArgs +from sglang.test.boundary_fixtures import stub_plan, stub_stage from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator @@ -50,12 +59,11 @@ def _owned_input(hidden, residual, batch=_DECODE): def _communicator(): - comm = LayerCommunicator.__new__(LayerCommunicator) + comm = stub_plan() comm.fusions = CuteDSLFusion() - comm.input_layernorm = RMSNorm(8, eps=1e-6) - comm.post_attention_layernorm = RMSNorm(8, eps=1e-6) + comm.norm = RMSNorm(8, eps=1e-6) comm.enable_fused_ar_quant = False - comm._attn_input_fusions = comm._select_attn_input_fusions() + comm._attn_input_fusions = attention_fusions(comm) # Only the ordinary batches' attention input half, with these entries. comm._sp_steps = comm._input_scattered_steps = comm._cp_steps = None comm._steps = SimpleNamespace( @@ -85,6 +93,15 @@ def _bound(entry): return entry.func, entry.args +def _test_layer(**kwargs): + comm = _communicator() + return SimpleNamespace( + attn_stage=stub_stage(comm, StageKind.ATTENTION), + ffn_stage=stub_stage(comm, StageKind.FFN), + **kwargs, + ) + + def _install(layers, **kwargs): reset_context() publish(ServerArgs(model_path="dummy"), role="test") @@ -136,8 +153,8 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): lambda *a, **k: pytest.fail("fell through to the unfused path"), ), ): - out_hidden, _ = last.prepare_attn( - *_owned_input(hidden_states, torch.zeros(8, 8)) + out_hidden, _ = stub_stage(last, StageKind.ATTENTION)._prepare_input( + hidden_states, torch.zeros(8, 8), _DECODE ) assert torch.equal(out_hidden, torch.ones(8, 8)) @@ -156,25 +173,33 @@ def test_cutedsl_entries_come_before_the_base_fused_kernel(): fusion._reduce_output_and_update_and_read_residual, (comm,), ) - assert base_input == comm._reduce_output_and_update_and_read_residual - base = comm._mlp_input_reduce_output_and_update_and_read_residual + assert _bound(base_input) == (complete_attention_input, (comm,)) + base = (complete_ffn_input, (comm,)) cutedsl = (fusion._mlp_input_reduce_output_and_update_and_read_residual, (comm,)) with patch( f"{_MODULE}.get_parallel", return_value=SimpleNamespace(attn_tp_size=2, tp_size=2), ): - comm._declared = SimpleNamespace(input_rows=Layout(frozenset())) - fusions = comm._select_mlp_input_fusions() + comm.variants = { + BatchVariant.ORDINARY: SimpleNamespace( + incoming=SimpleNamespace(residual=Layout(frozenset())) + ) + } + fusions = ffn_fusions(comm) assert _bound(fusions[0].run) == cutedsl - assert fusions[1].run == base + assert _bound(fusions[1].run) == base # The workspace reduces over the TP group, which is the attention-TP group here. assert [f.completes for f in fusions] == [SumGroup.ATTN_TP] * 2 # A residual on each rank's slice is gathered first, which the # workspace does not do. - comm._declared = SimpleNamespace( - input_rows=Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) - ) - assert [f.run for f in comm._select_mlp_input_fusions()] == [base] + comm.variants = { + BatchVariant.ORDINARY: SimpleNamespace( + incoming=SimpleNamespace( + residual=Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) + ) + ) + } + assert [_bound(f.run) for f in ffn_fusions(comm)] == [base] def test_the_fusion_runs_only_on_the_ffn_full_rows(): @@ -204,7 +229,7 @@ def test_the_fusion_runs_only_on_the_ffn_full_rows(): (frozenset({TokenAxis.ATTN_TP_SCATTER}), False), ): comm._batch_steps = lambda fb, rows=Layout(sharded): SimpleNamespace( - ffn=SimpleNamespace(input_rows=rows) + attention=None, ffn=SimpleNamespace(input_rows=rows) ) assert fusion._common_eligible(comm, _DECODE, 8) is eligible @@ -226,12 +251,18 @@ def test_install_does_not_require_a_fused_successor(): reset_context() publish(ServerArgs(model_path="dummy"), role="test") first, last = _communicator(), _communicator() - ordinary = SimpleNamespace(layer_communicator=SimpleNamespace(fusions=None)) + ordinary = SimpleNamespace(ffn_stage=SimpleNamespace(fusions=None)) install_cutedsl_fusion( [ - SimpleNamespace(layer_communicator=first), + SimpleNamespace( + attn_stage=stub_stage(first, StageKind.ATTENTION), + ffn_stage=stub_stage(first, StageKind.FFN), + ), ordinary, - SimpleNamespace(layer_communicator=last), + SimpleNamespace( + attn_stage=stub_stage(last, StageKind.ATTENTION), + ffn_stage=stub_stage(last, StageKind.FFN), + ), ], hidden_size=8, top_k=2, @@ -244,12 +275,9 @@ def test_install_does_not_require_a_fused_successor(): def test_a_replicated_output_producer_keeps_its_own_all_reduce(eligible): - layers = [ - SimpleNamespace(layer_communicator=_communicator(), replicated=value) - for value in (True, False) - ] + layers = [_test_layer(replicated=value) for value in (True, False)] _install(layers, requires_local_reduction=lambda layer: layer.replicated) - replicated, plain = [layer.layer_communicator for layer in layers] + replicated, plain = [layer.ffn_stage.plan for layer in layers] assert replicated.fusions.requires_local_reduction assert not plain.fusions.requires_local_reduction # The restriction belongs to the producer; incoming sums can still fuse. @@ -261,13 +289,15 @@ def test_a_service_nested_under_a_wrapper_is_prepared(): an unprepared service declines every M, leaving no fusion at all.""" prepared = [] layer = torch.nn.Linear(2, 2) - layer.layer_communicator = _communicator() + comm = _communicator() + layer.attn_stage, layer.ffn_stage = ( + stub_stage(comm, StageKind.ATTENTION), + stub_stage(comm, StageKind.FFN), + ) wrapper = torch.nn.Module() wrapper.language_model = torch.nn.Sequential(layer) _install([layer]) - layer.layer_communicator.fusions.service.prepare = lambda *, max_m: prepared.append( - max_m - ) + layer.ffn_stage.fusions.service.prepare = lambda *, max_m: prepared.append(max_m) # The workspace M bound is the largest of every framework source. reset_context() @@ -395,7 +425,7 @@ def test_a_handoff_the_kernel_does_not_take_is_completed_then_normed(): return residual * 2, residual comm = _communicator() - comm.input_layernorm = AddNorm() + comm.norm = AddNorm() comm.fusions.service = SimpleNamespace( finalize=lambda **kw: pytest.fail("the kernel does not take this batch") ) @@ -406,8 +436,8 @@ def test_a_handoff_the_kernel_does_not_take_is_completed_then_normed(): patch.object(CuteDSLFusion, "_should_use_finalize", return_value=False), patch.object(CuteDSLFusion, "_common_eligible", return_value=False), ): - hidden, residual = comm.prepare_attn( - *_owned_input(_handoff(finish), torch.full((2, 8), 2.0)) + hidden, residual = stub_stage(comm, StageKind.ATTENTION)._prepare_input( + _handoff(finish), torch.full((2, 8), 2.0), _DECODE ) finish.assert_called_once_with() torch.testing.assert_close(residual.residual, torch.full((2, 8), 3.0)) @@ -415,12 +445,11 @@ def test_a_handoff_the_kernel_does_not_take_is_completed_then_normed(): def test_the_layer_stack_hands_a_handoff_only_to_a_final_norm_that_takes_it(): - comm = LayerCommunicator.__new__(LayerCommunicator) for takes, completed in ((True, 0), (False, 1)): finish = MagicMock(return_value=torch.ones(2, 8)) handoff = _handoff(finish) - hidden, _ = comm.finish_layer_stack( - *_owned_input(handoff, torch.zeros(2, 8)), final_norm_takes_handoff=takes + hidden, _ = finish_layer_stack( + handoff, torch.zeros(2, 8), _DECODE, final_norm_takes_handoff=takes ) assert finish.call_count == completed assert (hidden is handoff) == takes diff --git a/test/registered/unit/layers/test_batch_owned_residual.py b/test/registered/unit/layers/test_batch_owned_residual.py index fc5967a354..5f061daf16 100644 --- a/test/registered/unit/layers/test_batch_owned_residual.py +++ b/test/registered/unit/layers/test_batch_owned_residual.py @@ -10,24 +10,12 @@ from unittest.mock import Mock, patch import torch from sglang.srt.batch_overlap.two_batch_overlap import TboForwardBatchPreparer -from sglang.srt.layers.communicator import ( - ADD, - BoundarySteps, - EdgeDecl, - LayerCommunicator, - Layout, - StageEntry, - StageInput, - StageOutput, - make_boundary, -) -from sglang.srt.layers.communicator.legacy_stage import StageCommunicator +from sglang.srt.layers.communicator import ADD from sglang.srt.layers.communicator.output import UnreducedOutput from sglang.srt.layers.communicator.residual import batch from sglang.srt.model_executor.forward_batch_info import ( ForwardBatch, ForwardMode, - PPProxyTensors, ) from sglang.srt.runtime_context import get_context, get_parallel from sglang.test.ci.ci_register import register_cpu_ci @@ -41,13 +29,15 @@ class TestBatchOwnedResidual(CustomTestCase): fb = SimpleNamespace(residual_stream=None) with self.assertRaisesRegex(RuntimeError, "start"): batch.current(fb) - old = batch.start(fb) + batch.start(fb) + old = batch.current(fb) self.assertIs(old, batch.current(fb)) value = torch.ones(2, 4) ref = weakref.ref(value) old.write(value) del value, old - fresh = batch.start(fb) + batch.start(fb) + fresh = batch.current(fb) self.assertIs(fresh, batch.current(fb)) self.assertIsNone(fresh.pending) self.assertIsNone(fresh.residual) @@ -55,41 +45,31 @@ class TestBatchOwnedResidual(CustomTestCase): def test_terminal_completion_releases_batch_before_final_norm(self): fb = SimpleNamespace(residual_stream=None) - stream = batch.start(fb) + batch.start(fb) + stream = batch.current(fb) residual = torch.full((2, 4), 3.0) partial = torch.ones(2, 4) group = SimpleNamespace(all_reduce=Mock(side_effect=lambda x: x * 2)) stream.write(residual) hidden = stream.leave(UnreducedOutput(partial, group=group), ADD) - complete, residual_value = LayerCommunicator.finish_layer_stack( - hidden, stream, fb - ) + result = batch.norm(hidden, fb, lambda value, prior: (value + prior, prior)) self.assertIsNone(fb.residual_stream) - self.assertIs(residual_value, residual) group.all_reduce.assert_called_once() - torch.testing.assert_close(complete + residual_value, torch.full((2, 4), 5.0)) + torch.testing.assert_close(result, torch.full((2, 4), 5.0)) with self.assertRaisesRegex(RuntimeError, "start"): batch.current(fb) - def test_pp_receive_attaches_and_export_releases_the_same_stream(self): - communicator = LayerCommunicator.__new__(LayerCommunicator) - communicator._batch_steps = lambda fb: SimpleNamespace( - attention=SimpleNamespace(input_sum=None), ffn=None - ) + def test_pp_export_releases_the_batch_owner(self): fb = SimpleNamespace(residual_stream=None) + batch.start(fb) hidden = torch.full((2, 4), 2.0) residual = torch.ones(2, 4) - received, stream = communicator.from_pp( - PPProxyTensors({"hidden_states": hidden, "residual": residual}), fb - ) - self.assertIs(stream, batch.current(fb)) - self.assertIs(stream.pending.value, received) - self.assertIs(stream.pending.update, ADD) - exported, exported_residual = communicator.finish_layer_stack( - received, stream, fb - ) - self.assertIs(exported, hidden) - self.assertIs(exported_residual, residual) + stream = batch.current(fb) + stream.write(residual) + output = stream.leave(hidden, ADD) + proxy = batch.to_pp(output, fb) + self.assertIs(proxy["hidden_states"], hidden) + self.assertIs(proxy["residual"], residual) self.assertIsNone(fb.residual_stream) def test_tbo_metadata_and_reused_batch_remain_stream_free_between_calls(self): @@ -127,10 +107,11 @@ class TestBatchOwnedResidual(CustomTestCase): for start in (0, 2) ] self.assertTrue(all(child.residual_stream is None for child in children)) - owner = batch.start(parent) + batch.start(parent) + owner = batch.current(parent) self.assertTrue(all(child.residual_stream is None for child in children)) hidden = owner.write(torch.ones(size, 4)) - LayerCommunicator.finish_layer_stack(hidden, owner, parent) + batch.norm(hidden, parent, lambda value: value) cloned = replace(parent) self.assertIsNone(cloned.residual_stream) self.assertIsNone(parent.residual_stream) @@ -172,19 +153,14 @@ class TestBatchOwnedResidual(CustomTestCase): group = SimpleNamespace(all_reduce=Mock(side_effect=lambda x: x * 2)) class Layer: - layer_communicator = LayerCommunicator.__new__(LayerCommunicator) - - def __call__(self, positions, hidden, forward_batch, *legacy_residual): + def __call__(self, positions, hidden, forward_batch, **kwargs): stream = batch.current(forward_batch) - if legacy_residual: - self_outer.assertIs(legacy_residual[0], stream) owner_ids.append(id(stream)) hidden, old = stream.finish(hidden) stream.write(hidden if old is None else hidden + old) - result = stream.leave( + return stream.leave( UnreducedOutput(torch.ones_like(hidden), group=group), ADD ) - return (result, stream) if legacy_residual else result self_outer = self fb = SimpleNamespace( @@ -253,51 +229,6 @@ class TestBatchOwnedResidual(CustomTestCase): self.assertIs(batch.take_output(value, fb), value) self.assertIsNone(fb.residual_stream) - def test_interleaved_stages_share_only_their_own_batch_stream(self): - class Read: - norms_plainly = False - - def enter(self, value): - return value - - def read(self, value, norm, quant_format="", **kwargs): - return value * 2, value - - def update_and_read(self, update, value, residual, norm, **kwargs): - updated = update.update(value, residual) - return updated * 2, updated - - rows = Layout(frozenset()) - boundary = make_boundary( - EdgeDecl(StageOutput(rows), StageInput(rows, read=Read()), rows, rows) - ) - steps = BoundarySteps( - StageEntry(boundary.prepare, rows), None, StageOutput(rows), None, False - ) - stage = StageCommunicator( - SimpleNamespace(input_layernorm=None), - "attention", - "input_layernorm", - ) - a, b = [SimpleNamespace(residual_stream=None) for _ in range(2)] - sa, sb = batch.start(a), batch.start(b) - first, first_alias = stage._prepare(torch.ones(2, 4), sa, a, steps) - second, second_alias = stage._prepare(torch.full((2, 4), 10.0), sb, b, steps) - self.assertIs(first_alias, batch.current(a)) - self.assertIs(second_alias, batch.current(b)) - first = sa.leave(first * 3, ADD) - second = sb.leave(second * 5, ADD) - torch.testing.assert_close( - stage._prepare(first, sa, a, steps)[0], torch.full((2, 4), 14.0) - ) - torch.testing.assert_close( - stage._prepare(second, sb, b, steps)[0], torch.full((2, 4), 220.0) - ) - self.assertIsNot(batch.current(a), batch.current(b)) - batch.start(a) - self.assertIsNone(batch.current(a).residual) - self.assertIsNotNone(batch.current(b).residual) - if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index fe81ffd979..c732c09895 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -2083,7 +2083,7 @@ class TestTwoLayers(CustomTestCase): ) def forward(rank): - batch = SimpleNamespace(**vars(forward_batch), residual_stream=None) + batch = SimpleNamespace(**vars(forward_batch)) s = states[rank] layers = [ make_test_stages( diff --git a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py index 4531203d2c..c0f766190c 100644 --- a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py +++ b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py @@ -261,8 +261,9 @@ class TestLongcatNextnReducesItsMlp(CustomTestCase): ), patch.object(nextn, "RMSNorm"), patch.object(nextn, "get_parallel"), - patch.object(nextn, "LayerFacts"), - patch.object(nextn, "LayerCommunicator"), + patch.object(nextn, "declare_attn"), + patch.object(nextn, "declare_ffn"), + patch.object(nextn, "make_stages", return_value=(None, None)), ): nextn.LongcatFlashDenseDecoderLayer(config, layer_id=0) self.assertIs(mlp_kwargs.get("reduce_results"), True) diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index 64c2345259..3cdc3ec5b8 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -7,6 +7,8 @@ import torch from torch import nn from sglang.srt.layers import communicator as comm +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList +from sglang.srt.layers.communicator import StageKind from sglang.srt.layers.communicator import ops as comm_ops from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.model_executor.forward_batch_info import ForwardMode, PPProxyTensors @@ -21,6 +23,7 @@ from sglang.srt.models.llama4 import Llama4Model from sglang.srt.models.qwen3 import Qwen3Model from sglang.srt.models.qwen3_vl import Qwen3LLMModel from sglang.srt.models.qwen3_vl_moe import Qwen3MoeLLMModel +from sglang.test.boundary_fixtures import stub_plan, stub_stage from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -53,9 +56,9 @@ class DeferringLayer(nn.Module): super().__init__() self.return_topk = return_topk self.stage_api = stage_api - self.layer_communicator = comm.LayerCommunicator.__new__(comm.LayerCommunicator) - self.layer_communicator.input_layernorm = None - self.attn_stage = self.layer_communicator.attn + self.layer_communicator = stub_plan() + self.layer_communicator.norm = None + self.attn_stage = stub_stage(self.layer_communicator, StageKind.ATTENTION) self.layer_communicator._steps = comm.BoundarySteps( attention=comm.StageEntry( prepare=partial( @@ -91,16 +94,18 @@ class DeferringLayer(nn.Module): ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial, ffn_sum_is_movable=True, ) - self.layer_communicator._ffn_sum_moves_to_next_layer = lambda batch, **_: defer + self.layer_communicator.output._ffn_sum_moves_to_next_layer = ( + lambda batch, **_: defer + ) self.layer_communicator.is_last_layer = False self.layer_communicator._sp_steps = None self.layer_communicator._input_scattered_steps = None self.layer_communicator._cp_steps = None - self.layer_communicator.ffn_reduction_group = lambda forward_batch: GROUP - self.layer_communicator._ffn_leaves_sum_to_reduce_scatter = ( + self.layer_communicator.output.ffn_reduction_group = lambda forward_batch: GROUP + self.layer_communicator.output._ffn_leaves_sum_to_reduce_scatter = ( lambda batch, dp_step: False ) - self.layer_communicator._complete_ffn_output_now = ( + self.layer_communicator.output._complete_ffn_output_now = ( lambda hidden, residual, **_: (all_reduce(hidden), residual) ) @@ -127,7 +132,7 @@ class DeferringLayer(nn.Module): capture = kwargs.get("capture_output") if capture is not None: capture(residual.clone()) - with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + with self.layer_communicator.output.ffn_exit(forward_batch) as ffn_exit: partial = torch.full_like(hidden_states, 0.5) if stream is not None: stream.write(residual) @@ -175,6 +180,11 @@ def build_model(model_cls, *, defer, capture): Llama4Model, Qwen3Model, Qwen3LLMModel, + Glm4MoeModel, + Glm4MoeLiteModel, + LagunaModel, + Qwen3MoeLLMModel, + Glm5NextModel, ), ) for i in range(NUM_LAYERS) @@ -185,19 +195,24 @@ def build_model(model_cls, *, defer, capture): class TestAuxCaptureDeferredAllreduce(CustomTestCase): def test_skipping_an_empty_capture_does_not_complete_its_output(self): - boundary = comm.LayerCommunicator.__new__(comm.LayerCommunicator) + boundary = stub_plan() + boundary.norm = None partial = torch.empty(0, 4) - owed = comm.UnreducedOutput(partial, group=GROUP) + batch = SimpleNamespace(residual_stream=ResidualStream(partial)) + owed = batch.residual_stream.leave( + comm.UnreducedOutput(partial, group=GROUP), comm.ADD + ) with patch.object(GROUP, "all_reduce") as reduce: - hidden, captured = boundary.capture_output(owed, partial, skip_empty=True) + hidden, captured = stub_stage(boundary, StageKind.ATTENTION).capture_output( + owed, batch, skip_empty=True + ) self.assertIs(hidden, owed) self.assertIsNone(captured) reduce.assert_not_called() def test_snapshot_without_a_residual_does_not_alias_the_main_output(self): - boundary = comm.LayerCommunicator.__new__(comm.LayerCommunicator) hidden = torch.ones(2, 4) - captured = boundary.snapshot(hidden, None) + captured = ResidualStream().snapshot(hidden) hidden.zero_() torch.testing.assert_close(captured, torch.ones_like(hidden)) @@ -235,6 +250,49 @@ class TestAuxCaptureDeferredAllreduce(CustomTestCase): reduce.call_count, NUM_LAYERS - 1 if defer else 0 ) + def test_glm_capture_owns_contracted_and_multi_rank_gathered_outputs(self): + inputs = torch.ones(2, 4) + original_capture = AuxHiddenStateList.capture + for mhc in (False, True): + for size in (1, 2): + with self.subTest(mhc=mhc, size=size): + ownership = [] + + def capture(collector, value, *, owned=False): + ownership.append(owned) + original_capture(collector, value, owned=owned) + + group = SimpleNamespace( + world_size=size, + all_gather=lambda value, dim: ( + value if size == 1 else torch.cat([value] * size, dim=dim) + ), + ) + model = build_model(Glm5NextModel, defer=True, capture=True) + model.enable_a2a_moe = True + model.config = SimpleNamespace(mhc=mhc, hc_mult=2) + batch = SimpleNamespace( + can_run_tbo=False, + forward_mode=ForwardMode.DECODE, + capture_hidden_mode=SimpleNamespace(need_capture=lambda: True), + ) + with ( + patch.object(AuxHiddenStateList, "capture", capture), + patch( + "sglang.srt.models.glm5_next.get_parallel", + return_value=SimpleNamespace(attn_tp_group=group), + ), + ): + _, snapshots = model( + None, None, batch, input_embeds=inputs.clone() + ) + self.assertEqual(ownership, [mhc or size > 1] * 3) + for boundary, value in enumerate(snapshots, 1): + torch.testing.assert_close( + value, + torch.full((2 * size, 2 if mhc else 4), 1.0 + boundary), + ) + def test_terminal_capture_uses_the_final_norm_boundary(self): inputs = torch.ones(2, 4) batch = SimpleNamespace( @@ -309,42 +367,45 @@ class TestPipelineResidualReception(CustomTestCase): torch.testing.assert_close(result, torch.full_like(inputs, 2.25)) def test_written_streams_do_not_read_a_separate_residual(self): - comm_instance = comm.LayerCommunicator.__new__(comm.LayerCommunicator) - comm_instance._batch_steps = lambda fb: SimpleNamespace( - attention=SimpleNamespace(input_sum=None), ffn=None - ) - comm_instance._residual = SimpleNamespace( - ffn_update=SimpleNamespace(at_producer=True) + comm_instance = stub_plan() + comm_instance.norm = None + comm_instance._batch_steps = lambda batch: SimpleNamespace( + attention=SimpleNamespace(input_sum=None) ) + batch = SimpleNamespace(residual_stream=None) + comm_instance.residual_in_hidden = True streams = torch.randn(2, 4, 3) - hidden, residual = comm_instance.from_pp( - PPProxyTensors({"hidden_states": streams}), - SimpleNamespace(residual_stream=None), + hidden = stub_stage(comm_instance, StageKind.ATTENTION).from_pp( + PPProxyTensors({"hidden_states": streams}), batch ) + residual = batch.residual_stream self.assertIs(hidden, streams) self.assertIsNone(residual.pending) self.assertIs(residual.residual, hidden) def test_optional_residual_and_declared_partial_keep_the_wire_values(self): - comm_instance = comm.LayerCommunicator.__new__(comm.LayerCommunicator) - comm_instance._batch_steps = lambda fb: SimpleNamespace( - attention=SimpleNamespace(input_sum=None), ffn=None + comm_instance = stub_plan() + comm_instance.norm = None + comm_instance._batch_steps = lambda batch: SimpleNamespace( + attention=SimpleNamespace(input_sum=None) ) + batch = SimpleNamespace(residual_stream=None) partial = torch.randn(2, 4) prior = torch.randn_like(partial) - hidden, residual = comm_instance.from_pp( - PPProxyTensors({"hidden_states": partial, "residual": prior}), - SimpleNamespace(residual_stream=None), + hidden = stub_stage(comm_instance, StageKind.ATTENTION).from_pp( + PPProxyTensors({"hidden_states": partial, "residual": prior}), batch ) + residual = batch.residual_stream self.assertIs(hidden, partial) self.assertIs(residual.residual, prior) self.assertIs(residual.pending.value, partial) missing = PPProxyTensors({"hidden_states": partial}) with self.assertRaises(KeyError): - comm_instance.from_pp(missing, SimpleNamespace(residual_stream=None)) - hidden, residual = comm_instance.from_pp( - missing, SimpleNamespace(residual_stream=None), allow_missing_residual=True + stub_stage(comm_instance, StageKind.ATTENTION).from_pp(missing, batch) + hidden = stub_stage(comm_instance, StageKind.ATTENTION).from_pp( + missing, batch, allow_missing_residual=True ) + residual = batch.residual_stream self.assertIs(hidden, partial) self.assertIsNone(residual.pending) self.assertIs(residual.residual, hidden) diff --git a/test/registered/unit/models/test_deepstack_deferred_allreduce.py b/test/registered/unit/models/test_deepstack_deferred_allreduce.py index 78340fa056..f7fa0c57ed 100644 --- a/test/registered/unit/models/test_deepstack_deferred_allreduce.py +++ b/test/registered/unit/models/test_deepstack_deferred_allreduce.py @@ -1,5 +1,3 @@ -from sglang.srt.layers.communicator.residual.add_norm import ADD - """Deepstack visual embeddings are added to a decoder layer's output. When the layer left its FFN all-reduce to the next layer, that output is one rank's partial sum, and an embedding added to it is counted once per rank.""" @@ -10,8 +8,10 @@ from types import SimpleNamespace import torch from torch import nn -from sglang.srt.layers.communicator import LayerCommunicator, UnreducedOutput +from sglang.srt.layers.communicator import UnreducedOutput from sglang.srt.layers.communicator.residual import batch as residual_batch +from sglang.srt.layers.communicator.residual.add_norm import ADD +from sglang.srt.layers.communicator.residual.stream import OwedOutput from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -41,21 +41,19 @@ class DeferringLayer(nn.Module): super().__init__() self.is_last_layer = is_last_layer # The model ends its layers at this communicator's finish_layer_stack. - self.layer_communicator = LayerCommunicator.__new__(LayerCommunicator) def forward( self, positions=None, hidden_states=None, forward_batch=None, residual=None, **_ ): - stream = forward_batch.residual_stream - hidden_states, residual = stream.finish(hidden_states) + if isinstance(hidden_states, UnreducedOutput): + hidden_states = all_reduce(hidden_states.partial) residual = hidden_states if residual is None else hidden_states + residual - stream.write(residual) - output = ( - torch.ones_like(residual) - if self.is_last_layer - else UnreducedOutput(torch.full_like(residual, 1 / TP_SIZE), group=GROUP) + if self.is_last_layer: + return torch.ones_like(residual), residual + return ( + UnreducedOutput(torch.full_like(residual, 1 / TP_SIZE), group=GROUP), + residual, ) - return stream.leave(output, ADD), stream class DeferringStageLayer(DeferringLayer): @@ -82,7 +80,8 @@ class SumNorm(nn.Module): def stub_model(cls, **attrs): model = cls.__new__(cls) nn.Module.__init__(model) - layers = [DeferringLayer(i == NUM_LAYERS - 1) for i in range(NUM_LAYERS)] + layer_type = DeferringStageLayer + layers = [layer_type(i == NUM_LAYERS - 1) for i in range(NUM_LAYERS)] common = dict( pp_group=SimpleNamespace(is_first_rank=True, is_last_rank=True), layers=layers, @@ -118,7 +117,13 @@ def qwen3_5(): ) -MODELS = (qwen3_vl_moe, qwen3_5) +def interns2_mobius(): + from sglang.srt.models.interns2_mobius import InternS2MobiusForCausalLM + + return stub_model(InternS2MobiusForCausalLM, meta_mlp=nn.ModuleList()) + + +MODELS = (qwen3_vl_moe, qwen3_5, interns2_mobius) class TestDeepstackOnDeferredReduction(CustomTestCase): @@ -130,7 +135,7 @@ class TestDeepstackOnDeferredReduction(CustomTestCase): return model.forward( input_ids=None, positions=None, - forward_batch=SimpleNamespace(), + forward_batch=SimpleNamespace(residual_stream=None), input_embeds=self.embeds.clone(), input_deepstack_embeds=deepstack, ) @@ -174,10 +179,7 @@ class TestSplitPrefillCompletion(CustomTestCase): with self.subTest(model=model_cls.__name__, tokens=tokens): # Also leave the final output: the stack exit must handle it # independently of the last layer's fusion decision. - stage_api = model_cls in ( - Qwen3ForCausalLM, - SarvamMoEForCausalLM, - ) + stage_api = True layer_type = DeferringStageLayer if stage_api else DeferringLayer wrapper = SimpleNamespace( model=SimpleNamespace( @@ -198,7 +200,10 @@ class TestSplitPrefillCompletion(CustomTestCase): input_embeds=torch.zeros(tokens, HIDDEN), ) self.assertIsNone(first) - self.assertIsNotNone(batch.residual_stream.pending.owed) + self.assertIsInstance( + batch.hidden_states, + OwedOutput if stage_api else UnreducedOutput, + ) result = model_cls.forward_split_prefill( wrapper, None, None, batch, (2, NUM_LAYERS) ) diff --git a/test/registered/unit/models/test_glm5_next_dflash_capture.py b/test/registered/unit/models/test_glm5_next_dflash_capture.py index df4f27ff34..0e6d9afdf9 100644 --- a/test/registered/unit/models/test_glm5_next_dflash_capture.py +++ b/test/registered/unit/models/test_glm5_next_dflash_capture.py @@ -1,6 +1,6 @@ """Regression for DFLASH aux-hidden capture on mHC models. -GLM-5.3-Flash runs with mhc=True. MHCLayerCommunicator folds the residual +GLM-5.3-Flash runs with mhc=True. Its MHC boundary folds the residual into the widened hidden state and returns residual=None, so CUDA-graph capture used to crash on `hidden_states + residual`. DFLASH also has to contract that widened state back to the draft hidden size; skipping the @@ -13,7 +13,7 @@ from types import SimpleNamespace import torch from torch import nn -from sglang.srt.layers.communicator import LayerCommunicator +from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.models.glm5_next import Glm5NextModel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -30,8 +30,7 @@ class TestGlm5NextDflashCapture(CustomTestCase): hidden_states = torch.arange(24, dtype=torch.float32).reshape(2, 12) - boundary = LayerCommunicator.__new__(LayerCommunicator) - _, captured = boundary.capture_output(hidden_states, None) + captured = ResidualStream().snapshot(hidden_states) actual = model._prepare_aux_hidden_state(captured) expected = hidden_states.unflatten(-1, (4, -1)).mean(dim=-2) diff --git a/test/registered/unit/models/test_last_layer_communicator.py b/test/registered/unit/models/test_last_layer_communicator.py index 2f213d28f3..a1a5da934b 100644 --- a/test/registered/unit/models/test_last_layer_communicator.py +++ b/test/registered/unit/models/test_last_layer_communicator.py @@ -175,15 +175,11 @@ def build(case, num_layers, layer_id, config=None, **kwargs): from sglang.srt.layers.communicator import declare_attn, declare_ffn - if hasattr(module, "make_stages"): - patches = dict( - declare_attn=declare_attn, - declare_ffn=declare_ffn, - make_stages=communicator, - ) - else: - facts = MagicMock() - patches = dict(LayerCommunicator=communicator, LayerFacts=facts) + patches = dict( + declare_attn=declare_attn, + declare_ffn=declare_ffn, + make_stages=communicator, + ) patches.update({name: recording_stub(name) for name in stubs}) if hasattr(module, "get_parallel"): patches["get_parallel"] = lambda: PARALLEL @@ -192,18 +188,6 @@ def build(case, num_layers, layer_id, config=None, **kwargs): config or make_config(num_layers), layer_id=layer_id, **kwargs ) communicator.assert_called_once() - if not hasattr(module, "make_stages"): - passed = communicator.call_args.kwargs - planned = facts.init_new.call_args.kwargs - return ( - passed, - { - "terminal": planned["layer_id"] == planned["num_layers"] - 1, - "enters_stack": planned["layer_id"] == 0, - "is_layer_sparse": planned["is_layer_sparse"], - }, - built, - ) declaration = communicator.call_args.args[1][0] sequence = communicator.call_args.kwargs return ( @@ -237,17 +221,6 @@ class TestTerminalStages(CustomTestCase): layer_id == NUM_LAYERS - 1, ) - def test_step3p5_dense_layers_never_defer_their_sum(self): - """A Step-3.5 dense layer's MLP all-reduces its own output unless - postprocess reduce-scatters it; its communicator is told it never leaves - the sum to the next layer. The MoE layers may.""" - for layer_id in range(NUM_LAYERS): - with self.subTest(layer_id=layer_id): - passed, planned, _ = build("step3p5", NUM_LAYERS, layer_id) - self.assertIs( - passed["allow_deferred_ffn_reduction"], planned["is_layer_sparse"] - ) - def test_draft_model_layer_is_last(self): """The single decoder layer of a NextN / MTP draft model is marked last, whatever layer id and layer count it is built with.""" diff --git a/test/registered/unit/models/test_layer_stack_exit.py b/test/registered/unit/models/test_layer_stack_exit.py index cffcbaf825..d799f05bbd 100644 --- a/test/registered/unit/models/test_layer_stack_exit.py +++ b/test/registered/unit/models/test_layer_stack_exit.py @@ -102,9 +102,6 @@ class Census: and isinstance(call.func, ast.Name) and call.func.id in { - "LayerCommunicator", - "MHCLayerCommunicator", - "make_decoder_stages", "make_stages", } for call in ast.walk(node) @@ -296,7 +293,7 @@ class TestLayerStackExit(CustomTestCase): and isinstance(call.func, ast.Attribute) and isinstance(call.func.value, ast.Name) and call.func.value.id == "residual_batch" - and call.func.attr in {"norm", "to_pp", "fold", "finish"} + and call.func.attr in {"norm", "to_pp", "finish"} ] if batch_exits and all( not inside_loop(forward, call) for call in batch_exits diff --git a/test/registered/unit/models/test_longcat_flash_shortcut.py b/test/registered/unit/models/test_longcat_flash_shortcut.py index 33f491a7c3..2bf9d71717 100644 --- a/test/registered/unit/models/test_longcat_flash_shortcut.py +++ b/test/registered/unit/models/test_longcat_flash_shortcut.py @@ -1,11 +1,13 @@ import unittest from types import SimpleNamespace -from unittest.mock import Mock import torch from torch import nn from sglang.srt.layers import communicator as comm +from sglang.srt.layers.communicator.adapters import branch +from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers.communicator.stage import StageCommunicator from sglang.srt.models.longcat_flash import LongcatFlashDecoderLayer from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator @@ -25,8 +27,8 @@ class TestLongcatShortcut(CustomTestCase): attention = comm.Layout(frozenset()) class Communicator(SimpleNamespace): - branch_output = comm.LayerCommunicator.branch_output - merge_branch = comm.LayerCommunicator.merge_branch + branch_output = branch.branch_output + merge_branch = branch.merge_branch fork_hidden = torch.full((rows // tp, 3), 2.0) fork_residual = torch.full_like(fork_hidden, 5.0) @@ -40,17 +42,27 @@ class TestLongcatShortcut(CustomTestCase): ) layer = LongcatFlashDecoderLayer.__new__(LongcatFlashDecoderLayer) nn.Module.__init__(layer) - layer.moe_layer_communicator = moe_communicator - layer.mlp_layer_communicator = [None, dense_communicator] + batch = SimpleNamespace(residual_stream=ResidualStream()) + layer.attn_stage = SimpleNamespace( + prepare=lambda h, fb: h, finish=lambda h, fb: h + ) + moe_communicator.post_attention_layernorm = None + dense_communicator.post_attention_layernorm = None + layer.moe_stage = StageCommunicator(moe_communicator, comm.StageKind.FFN, None) + layer.moe_stage.prepare = lambda h, fb: fork_hidden + layer.second_ffn = StageCommunicator( + dense_communicator, comm.StageKind.FFN, None + ) layer.self_attn = [lambda **kw: kw["hidden_states"]] layer.mlp = nn.Identity() - layer.forward_mlp = Mock( - return_value=( - torch.full((rows, 3), 3.0), - torch.full((rows, 3), 11.0), - None, - ) - ) + + def dense_branch(*args): + stream = ResidualStream(torch.full((rows, 3), 11.0)) + hidden = stream.leave(torch.full((rows, 3), 3.0), comm.ADD) + batch.residual_stream = stream + return hidden, None + + layer.forward_mlp = dense_branch with ( patch_communicator( "get_local_dp_buffer", @@ -65,9 +77,10 @@ class TestLongcatShortcut(CustomTestCase): return_value=SimpleNamespace(attn_tp_group=object()), ), ): - hidden, residual, _ = layer( - torch.arange(rows), torch.zeros(rows, 3), None, None, None, None + hidden, _ = layer( + torch.arange(rows), torch.zeros(rows, 3), batch, None, None ) + hidden, residual = batch.residual_stream.finish(hidden) torch.testing.assert_close(hidden, torch.full((rows, 3), 5.0)) torch.testing.assert_close(hidden + residual, torch.full((rows, 3), 16.0)) torch.testing.assert_close(fork_residual, torch.full_like(fork_residual, 5.0)) diff --git a/test/registered/unit/models/test_nemotron_h_stages.py b/test/registered/unit/models/test_nemotron_h_stages.py index 033cbe1f03..fdb689c9e3 100644 --- a/test/registered/unit/models/test_nemotron_h_stages.py +++ b/test/registered/unit/models/test_nemotron_h_stages.py @@ -1,7 +1,8 @@ import itertools import unittest +from dataclasses import replace from types import SimpleNamespace -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import torch @@ -10,15 +11,12 @@ from sglang.srt.layers.communicator import ( NORM_QUANT_READ, NORM_READ, Layout, - StageDecl, - StageInput, + MixerExit, StageOutput, SumGroup, TokenAxis, UnreducedOutput, - stage_edges, ) -from sglang.srt.layers.communicator.layer import MixerExit from sglang.srt.layers.moe.utils import should_skip_mlp_all_reduce from sglang.srt.models import nemotron_h_utils as utils from sglang.srt.runtime_context import get_parallel @@ -28,6 +26,27 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=3, suite="base-a-test-cpu") +def layer_stage(pattern, index): + from sglang.srt.layers.communicator.construction import BatchVariant + from sglang.srt.layers.communicator.factories import _connections + + previous = utils._declaration(pattern, index - 1) if index else None + declaration = replace( + utils._declaration(pattern, index), + previous=previous, + terminal=index == len(pattern) - 1, + ) + incoming, outgoing = _connections(declaration) + return SimpleNamespace( + kind=declaration.kind, + edges=( + incoming.entries[BatchVariant.ORDINARY], + outgoing.exits[BatchVariant.ORDINARY], + ), + enters_stack=index == 0, + ) + + def sizes(*, dp=1, tp=1): return { TokenAxis.ATTN_DP: dp, @@ -37,12 +56,31 @@ def sizes(*, dp=1, tp=1): def stages(pattern, *, dp=1, tp=1, a2a=False): - backend = SimpleNamespace(is_none=lambda: not a2a) + from sglang.test.communicator_patch import patch_communicator + + parallel = SimpleNamespace( + attn_dp_size=dp, + attn_tp_size=tp, + attn_cp_size=1, + tp_size=dp * tp, + moe_dp_size=1, + moe_dense_tp_size=None, + ) with ( - patch.object(utils, "token_axis_sizes", return_value=sizes(dp=dp, tp=tp)), - patch.object(utils, "get_moe_a2a_backend", return_value=backend), + patch_communicator( + "get_exec", + return_value=SimpleNamespace( + comm=SimpleNamespace(boundary_reduction="rs+rsv"), + overlap=SimpleNamespace(enable_two_batch_overlap=False), + ), + ), + patch_communicator("token_axis_sizes", return_value=sizes(dp=dp, tp=tp)), + patch_communicator("get_parallel", return_value=parallel), + patch_communicator("is_moe_input_scattered_across_dp_ranks", return_value=a2a), + patch_communicator("enable_moe_dense_fully_dp", return_value=False), + patch_communicator("_generic_prefill_cp_shards_tokens", return_value=False), ): - return [utils.layer_stage(pattern, i) for i in range(len(pattern))] + return [layer_stage(pattern, i) for i in range(len(pattern))] class TestStageEdges(CustomTestCase): @@ -94,8 +132,8 @@ class TestStageEdges(CustomTestCase): "MM": (SumGroup.ATTN_TP, False, True), "-M": (SumGroup.TP, False, True), "EM": (SumGroup.MOE_OUTPUT, False, True), - "--": (None, False, False), - "E-": (None, False, False), + "--": (SumGroup.TP, False, True), + "E-": (SumGroup.MOE_OUTPUT, False, True), } for pattern, expected in cases.items(): with self.subTest(pattern=pattern): @@ -128,19 +166,6 @@ class TestStageEdges(CustomTestCase): ) self.assertIs(out_of.produced.update, ADD) - def test_the_residual_follows_the_input_onto_a_finer_slice(self): - axis_sizes = sizes(dp=2, tp=2) - attention = Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=axis_sizes) - local = Layout.sharded_over( - TokenAxis.ATTN_DP, TokenAxis.ATTN_TP_SCATTER, axis_sizes=axis_sizes - ) - full = Layout.sharded_over(axis_sizes=axis_sizes) - for need, during in ((local, local), (full, attention)): - stage = StageDecl(StageInput(need), StageOutput(need)) - into, out_of = stage_edges(previous=None, stage=stage, rows=attention) - self.assertEqual((into.residual, into.residual_to), (attention, during)) - self.assertEqual((out_of.residual, out_of.residual_to), (during, attention)) - class TestMixerExit(CustomTestCase): """A mixer skips its output all-reduce when its output always leaves the sum @@ -163,7 +188,9 @@ class TestMixerExit(CustomTestCase): leaves_for_next_layer=may, ) communicator = SimpleNamespace( - _batch_steps=lambda batch: SimpleNamespace(ffn_output=produced), + plan=SimpleNamespace( + _batch_steps=lambda batch: SimpleNamespace(ffn_output=produced), + ), _ffn_sum_can_move_to_next_layer=MagicMock(return_value=movable), ) hidden = torch.ones(2, 4) diff --git a/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py index e57e62e3c4..9022c45ee9 100644 --- a/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py +++ b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py @@ -1,4 +1,5 @@ -from sglang.srt.layers.communicator.layer import FfnExit +from sglang.srt.layers.communicator.residual import batch as residual_batch +from sglang.test.boundary_fixtures import stub_plan """A Step-3.5 dense layer must leave its all-reduce out when postprocess reduce-scatters its output; otherwise the output is summed twice. The layer's @@ -12,7 +13,7 @@ from unittest.mock import MagicMock import torch from sglang.srt.layers import communicator as comm -from sglang.srt.layers.communicator.residual import batch as residual_batch +from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.runtime_context import get_forward from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -33,24 +34,40 @@ def dense_layer(*, reduce_scatter): return hidden_states complete = MagicMock(side_effect=lambda h, r: (h, r)) - communicator = SimpleNamespace( - prepare_attn=lambda h, r, fb, **_: (h, h if r is None else r), - prepare_mlp=lambda h, r, fb: (h, r), - _batch_steps=lambda fb: SimpleNamespace( - ffn_output=SimpleNamespace(update=comm.ADD) + plan = stub_plan() + plan._batch_steps = lambda fb: SimpleNamespace( + ffn_output=comm.StageOutput(comm.Layout(frozenset())), + returns_over_dp=False, + ffn_output_move_completes_sum=False, + ) + plan.output._select_ffn_completion = lambda fb: comm.FfnCompletion( + defer_moe_finalize=False, + fuse_mlp_allreduce=False, + mlp_reduce_scatter=reduce_scatter, + complete=complete, + ) + + def stage_exit(fb): + result = plan.output.ffn_exit(fb) + result._stream = fb.residual_stream + return result + + object.__setattr__( + layer, + "attn_stage", + SimpleNamespace( + prepare=lambda h, fb, **kwargs: h, + finish=lambda h, fb: h, ), - _declared_ffn_sum=lambda steps, skipped: None, - _leave_ffn_output=lambda h, r, stream, update, declared_sum: (h, r), - # A dense layer never leaves its sum to the next layer. - _select_ffn_completion=lambda fb: comm.FfnCompletion( - defer_moe_finalize=False, - fuse_mlp_allreduce=False, - mlp_reduce_scatter=reduce_scatter, - complete=complete, + ) + object.__setattr__( + layer, + "ffn_stage", + SimpleNamespace( + prepare=lambda h, fb: h, + exit=stage_exit, ), ) - communicator.ffn_exit = lambda fb: FfnExit(communicator, fb) - object.__setattr__(layer, "layer_communicator", communicator) object.__setattr__(layer, "use_moe", False) object.__setattr__(layer, "mlp", mlp) object.__setattr__(layer, "self_attn", lambda **kwargs: kwargs["hidden_states"]) @@ -68,8 +85,9 @@ class TestStep3p5DenseReduceScatter(CustomTestCase): layer.forward( positions=None, hidden_states=torch.ones(2, 4), - forward_batch=batch, - residual=stream, + forward_batch=SimpleNamespace( + residual_stream=ResidualStream(torch.ones(2, 4)) + ), ) self.assertEqual(seen["mlp_reduce_scatter"], reduce_scatter) self.assertFalse(seen["fuse_mlp_allreduce"])