From f07c1a398efdf1772beb59916634c1338574ee91 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sun, 4 Oct 2026 16:38:36 -0700 Subject: [PATCH] [Refactor] Build the Qwen4 experimental decoders from stage boundaries (#42478) --- .../srt/arg_groups/boundary_reduction.py | 2 + .../srt/layers/layer_boundary/__init__.py | 4 + .../layers/layer_boundary/residual/gated.py | 198 ++++++++++ python/sglang/srt/models/qwen3_5.py | 122 ++++--- python/sglang/srt/models/qwen4_exp.py | 337 +++++++++--------- .../layer_boundary/test_gated_residual_ops.py | 256 +++++++++++++ .../test_moe_output_reduction.py | 154 ++++---- .../layer_boundary/test_qwen4_exp_ple_rows.py | 64 ++++ .../test_stage_producers_leave_sums.py | 19 +- test/registered/unit/test_model_overrides.py | 2 + 10 files changed, 854 insertions(+), 304 deletions(-) create mode 100644 python/sglang/srt/layers/layer_boundary/residual/gated.py create mode 100644 test/registered/unit/layer_boundary/test_gated_residual_ops.py create mode 100644 test/registered/unit/layer_boundary/test_qwen4_exp_ple_rows.py diff --git a/python/sglang/srt/arg_groups/boundary_reduction.py b/python/sglang/srt/arg_groups/boundary_reduction.py index d7ec8bc437..21ffc570c5 100644 --- a/python/sglang/srt/arg_groups/boundary_reduction.py +++ b/python/sglang/srt/arg_groups/boundary_reduction.py @@ -18,6 +18,8 @@ _AR_ARCHITECTURES = { "Qwen3ForRewardModel", "MossVLForCausalLM", "MossVLForConditionalGeneration", + "Qwen4ExpForConditionalGeneration", + "Qwen4ExpForCausalLMMTP", } diff --git a/python/sglang/srt/layers/layer_boundary/__init__.py b/python/sglang/srt/layers/layer_boundary/__init__.py index 456e6c02af..dffaa6ea30 100644 --- a/python/sglang/srt/layers/layer_boundary/__init__.py +++ b/python/sglang/srt/layers/layer_boundary/__init__.py @@ -72,6 +72,9 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import ( PLAIN_ADD, PLAIN_RESIDUAL_OPS, ) +from sglang.srt.layers.layer_boundary.residual.gated import ( + GatedResidualState, +) from sglang.srt.layers.layer_boundary.residual.ihc import ( IHCState, ) @@ -96,6 +99,7 @@ __all__ = [ "FfnExit", "FfnInputFusion", "DeferredFinalize", + "GatedResidualState", "IHCState", "LayerResidualOps", "Layout", diff --git a/python/sglang/srt/layers/layer_boundary/residual/gated.py b/python/sglang/srt/layers/layer_boundary/residual/gated.py new file mode 100644 index 0000000000..5b40bfaae1 --- /dev/null +++ b/python/sglang/srt/layers/layer_boundary/residual/gated.py @@ -0,0 +1,198 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Gated-injection hyper-connection residual streams.""" + +from dataclasses import dataclass +from typing import Callable, Optional + +import torch + +from sglang.srt.layers.layer_boundary.residual import LayerResidualOps +from sglang.srt.runtime_context import get_parallel + + +@dataclass +class GatedResidualState: + """A layer's residual as hc_count streams flattened into one tensor: a + stage reads its input by normalizing the streams and mixing them down with + per-stream gates, and writes its output back scaled by an injection + coefficient computed from that same normalized residual. + + Unlike the hyper-connection streams in mhc.py and ihc.py, whose read + produces the coefficient its write-back consumes, the coefficient here is + computed at write time. What has to survive from a stage's read to its + write-back is therefore the normalized residual itself, which this state + holds. Parameters belong to the owning layer's modules. + + Args: + expand: Widens the layer stack's input into the stream representation, + and passes an already widened residual through. + attn_mix: The attention stage's read. A layer whose own contribution + joins the streams before that read (the Qwen4 PLE embedding) folds + it in here, so this state does not need to know about it. + ffn_mix: The FFN stage's read. + attn_combine: Writes the attention output back into the streams. + ffn_combine: Writes the FFN output back into the streams. + """ + + expand: Callable + attn_mix: Callable + ffn_mix: Callable + attn_combine: Callable + ffn_combine: Callable + # Produced by a stage's read and consumed by that same stage's write-back. + normed: Optional[torch.Tensor] = None + + def _read(self, mix, residual, out_norm): + if out_norm is not None: + raise NotImplementedError( + "a gated hyper-connection read with a separate norm; the read " + "normalizes the streams itself" + ) + hidden_states, (residual, self.normed) = mix(residual) + return hidden_states, residual + + def read_attn_input(self, residual, out_norm=None): + return self._read(self.attn_mix, residual, out_norm) + + def update_and_read_ffn_input(self, hidden_states, residual, out_norm=None): + residual = self.apply_attn_combine(hidden_states, residual) + return self._read(self.ffn_mix, residual, out_norm) + + def apply_attn_combine(self, hidden_states, residual): + return self.attn_combine(hidden_states, (residual, self.normed)) + + def apply_ffn_combine(self, hidden_states, residual): + return self.ffn_combine(hidden_states, (residual, self.normed)) + + def clear_coefficients(self): + self.normed = None + + def slice_residual_attn_tp(self, residual): + parallel = get_parallel() + rank, size = parallel.attn_tp_rank, parallel.attn_tp_size + self.normed = self.normed.tensor_split(size)[rank] + return residual.tensor_split(size)[rank] + + def gather_residual_attn_tp(self, residual): + raise NotImplementedError( + "Unsupported: gated hyper-connection normed-residual allgather is " + "not implemented." + ) + + def residual_ops(self) -> LayerResidualOps: + return LayerResidualOps( + attn_readout=_AttnReadout(self), + attn_update=_AttnUpdate(self), + ffn_readout=_FfnReadout(self), + ffn_update=_FfnUpdate(self), + ) + + +class _AttnReadout: + """The normalized, gated mix of the streams, from streams that already hold + the previous layer's output: such a layer takes its input written back.""" + + is_plain_norm = False + reads_before_dp_gather = False + + def __init__(self, state: GatedResidualState): + self.state = state + + def init_residual(self, hidden_states): + return self.state.expand(hidden_states) + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + if quant_format: + raise NotImplementedError( + f"a gated hyper-connection attention input in {quant_format=}" + ) + return self.state.read_attn_input(residual, out_norm=norm) + + def update_and_read(self, update, hidden_states, residual, norm, **kwargs): + raise NotImplementedError( + "a gated hyper-connection layer takes its input written back" + ) + + +class _AttnUpdate: + """The gated injection of the attention output, using the normalized + residual its read produced. The injection is not a plain add, so it runs + only once the sum it writes in is complete.""" + + is_plain_add = False + applied_at_exit = False + outlives_layer = False + + def __init__(self, state: GatedResidualState): + self.state = state + + def update(self, hidden_states, residual): + return self.state.apply_attn_combine(hidden_states, residual) + + def slice_residual_attn_tp(self, residual): + return self.state.slice_residual_attn_tp(residual) + + def gather_residual_attn_tp(self, residual): + return self.state.gather_residual_attn_tp(residual) + + +class _FfnReadout: + """The attention output's injection and the FFN input's mix.""" + + is_plain_norm = False + reads_before_dp_gather = False + + def __init__(self, state: GatedResidualState): + self.state = state + + def init_residual(self, hidden_states): + return self.state.expand(hidden_states) + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + raise NotImplementedError( + "a gated hyper-connection FFN input read without its attention" + ) + + def update_and_read(self, update, hidden_states, residual, norm, **kwargs): + if not isinstance(update, _AttnUpdate) or update.state is not self.state: + raise NotImplementedError( + f"a gated hyper-connection FFN input after {update=}" + ) + return self.state.update_and_read_ffn_input( + hidden_states, residual, out_norm=norm + ) + + +class _FfnUpdate: + """The gated injection of the FFN output, which this layer runs itself. The + streams stay widened: the layer stack's terminal read mixes them down.""" + + is_plain_add = False + applied_at_exit = True + outlives_layer = False + + def __init__(self, state: GatedResidualState): + self.state = state + + def update(self, hidden_states, residual): + hidden_states = self.state.apply_ffn_combine(hidden_states, residual) + self.state.clear_coefficients() + return hidden_states + + def slice_residual_attn_tp(self, residual): + return self.state.slice_residual_attn_tp(residual) + + def gather_residual_attn_tp(self, residual): + return self.state.gather_residual_attn_tp(residual) diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 44cecb6b5b..a26ed8edf7 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -1041,9 +1041,6 @@ class Qwen3_5GatedDeltaNet(nn.Module): class Qwen3_5LinearDecoderLayer(nn.Module): """Qwen3.5 Decoder Layer with Linear Attention (GatedDeltaNet).""" - # True in a subclass built without stage boundaries. - _ffn_sums_itself = False - def __init__( self, config: Qwen3_5TextConfig, @@ -1052,6 +1049,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module): prefix: str = "", alt_stream: Optional[torch.cuda.Stream] = None, is_nextn: bool = False, + build_stages: bool = True, ) -> None: super().__init__() self.config = config @@ -1076,7 +1074,8 @@ class Qwen3_5LinearDecoderLayer(nn.Module): prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")), is_nextn=is_nextn, support_shared_expert_fusion=not _disable_shared_experts_fusion(), - reduce_results=self._ffn_sums_itself, + # The stage boundary completes this stage's sum. + reduce_results=False, ) is_layer_sparse = True is_previous_layer_sparse = True @@ -1088,7 +1087,8 @@ class Qwen3_5LinearDecoderLayer(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")), - reduce_results=self._ffn_sums_itself, + # The stage boundary completes this stage's sum. + reduce_results=False, ) _maybe_enable_silu_fp4_quant_fusion(self.mlp) is_layer_sparse = False @@ -1106,33 +1106,37 @@ class Qwen3_5LinearDecoderLayer(nn.Module): # it. Otherwise, stay on the plain AR+RMSNorm path. accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz) boundary_fusions = _layer_fusions(config, is_nextn) - self.attn_boundary, self.ffn_boundary = make_stages( - ( - declare_attn( - read=NormQuantReadout( - fp8_input=Fp8Input.TUPLE_AND_BF16 if accepts_fp8_input else None - ) + # A subclass that brings its own residual builds the stages itself. + if build_stages: + self.attn_boundary, self.ffn_boundary = make_stages( + ( + declare_attn( + read=NormQuantReadout( + fp8_input=Fp8Input.TUPLE_AND_BF16 + if accepts_fp8_input + else None + ) + ), + self.input_layernorm, + { + "fusions": boundary_fusions, + }, ), - self.input_layernorm, - { - "fusions": boundary_fusions, - }, - ), - ( - declare_ffn( - sparse=is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, + ( + declare_ffn( + sparse=is_layer_sparse, + next_layer_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + {"fusions": boundary_fusions}, ), - self.post_attention_layernorm, - {"fusions": boundary_fusions}, - ), - previous=declare_ffn( - sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) - if layer_id != 0 - else None, - terminal=layer_id == config.num_hidden_layers - 1, - ) def forward( self, @@ -1174,9 +1178,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module): class Qwen3_5AttentionDecoderLayer(nn.Module): """Qwen3.5 Decoder Layer with Full Attention.""" - # See Qwen3_5LinearDecoderLayer._ffn_sums_itself. - _ffn_sums_itself = False - def __init__( self, config: Qwen3_5TextConfig, @@ -1185,6 +1186,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): prefix: str = "", alt_stream: Optional[torch.cuda.Stream] = None, is_nextn: bool = False, + build_stages: bool = True, ) -> None: super().__init__() self.config = config @@ -1279,7 +1281,8 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix.replace(".self_attn", "")), - reduce_results=self._ffn_sums_itself, + # The stage boundary completes this stage's sum. + reduce_results=False, ) is_layer_sparse = False is_previous_layer_sparse = False @@ -1297,7 +1300,8 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): prefix=add_prefix("mlp", prefix.replace(".self_attn", "")), is_nextn=is_nextn, support_shared_expert_fusion=not _disable_shared_experts_fusion(), - reduce_results=self._ffn_sums_itself, + # The stage boundary completes this stage's sum. + reduce_results=False, ) is_layer_sparse = True is_previous_layer_sparse = True @@ -1317,33 +1321,35 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): # when qkv_proj can consume the returned quantized tuple. accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj) boundary_fusions = _layer_fusions(config, is_nextn) - self.attn_boundary, self.ffn_boundary = make_stages( - ( - declare_attn( - read=NormQuantReadout( - fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None - ) + # A subclass that brings its own residual builds the stages itself. + if build_stages: + self.attn_boundary, self.ffn_boundary = make_stages( + ( + declare_attn( + read=NormQuantReadout( + fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None + ) + ), + self.input_layernorm, + { + "fusions": boundary_fusions, + }, ), - self.input_layernorm, - { - "fusions": boundary_fusions, - }, - ), - ( - declare_ffn( - sparse=is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, + ( + declare_ffn( + sparse=is_layer_sparse, + next_layer_sparse=is_next_layer_sparse, + ), + self.post_attention_layernorm, + {"fusions": boundary_fusions}, ), - self.post_attention_layernorm, - {"fusions": boundary_fusions}, - ), - previous=declare_ffn( - sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse + previous=declare_ffn( + sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, ) - if layer_id != 0 - else None, - terminal=layer_id == config.num_hidden_layers - 1, - ) self.alt_stream = alt_stream diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py index 0894eac174..e8dd479358 100644 --- a/python/sglang/srt/models/qwen4_exp.py +++ b/python/sglang/srt/models/qwen4_exp.py @@ -26,13 +26,9 @@ from sglang.srt.layers.attention.linear.utils import ( ) from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.dp_attention import ( - attn_tp_all_gather, attn_tp_all_reduce, dp_gather_replicate, dp_scatter, - get_dp_global_num_tokens, - get_global_dp_buffer, - get_local_dp_buffer, is_allocation_symmetric, is_dp_attention_enabled, ) @@ -40,10 +36,17 @@ from sglang.srt.layers.hyperconnection import ( GatedResidual, HyperConnectionConfig, ) -from sglang.srt.layers.layer_boundary import get_attn_tp_context +from sglang.srt.layers.layer_boundary import ( + ExitRows, + GatedResidualState, + declare_attn, + declare_ffn, + get_attn_tp_context, + make_stages, +) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessorOutput -from sglang.srt.layers.moe import get_moe_a2a_backend, should_use_dp_reduce_scatterv from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.modelopt_quant import ( @@ -70,6 +73,7 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo get_tc_piecewise_forward_context, ) from sglang.srt.model_loader.weight_utils import default_weight_loader +from sglang.srt.models.qwen2_moe import Qwen2MoeSparseMoeBlock from sglang.srt.models.qwen3_5 import ( Qwen3_5AttentionDecoderLayer, Qwen3_5ForCausalLM, @@ -82,7 +86,7 @@ from sglang.srt.models.qwen4_exp_ple_table import ( make_ple_file_prefetcher, make_ple_file_rss_trimmer, ) -from sglang.srt.runtime_context import get_forward, get_parallel +from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import get_bool_env_var, is_hip, logger _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip() @@ -1409,10 +1413,64 @@ class Qwen4ExpPLELayer(nn.Module): return _pad_token_rows(output, batch.physical_tokens) -class Qwen4ExpLayerExtensionMixin: - # These layers drop the stage boundaries they inherit. - _ffn_sums_itself = True +class _MixStreams: + """The layer stack's terminal read: the gated mix down to one stream. The + stack ends on the mix, with no final norm after it.""" + def __init__(self, hyper_connection_mixer): + self.hyper_connection_mixer = hyper_connection_mixer + + def __call__(self, hidden_states): + return self.hyper_connection_mixer.mix(hidden_states)[0] + + +def _has_ple(layer_id, config) -> bool: + return (layer_id + 1) in config.ple_layer_ids + + +def _ffn_exit_rows(layer_id, config): + """A PLE layer's read adds an embedding computed for every row, so the FFN + before it hands its output on as full attention rows.""" + return ExitRows.ATTENTION if _has_ple(layer_id + 1, config) else None + + +def _build_qwen4_exp_stages(residual, *, sparse, layer_id, config): + """The attention and FFN stage boundaries of one gated hyper-connection layer. + + Each read normalizes the streams itself, so neither stage binds a norm, and + the writes are gated injections, so neither stage offers a fused + add-and-norm candidate. + """ + return make_stages( + ( + declare_attn(read=residual.attn_readout, update=residual.attn_update), + None, + ), + ( + declare_ffn( + sparse=sparse, + next_layer_sparse=sparse, + read=residual.ffn_readout, + update=residual.ffn_update, + exit_rows=_ffn_exit_rows(layer_id, config), + ), + None, + ), + previous=( + declare_ffn( + sparse=sparse, + next_layer_sparse=sparse, + update=residual.ffn_update, + exit_rows=_ffn_exit_rows(layer_id - 1, config), + ) + if layer_id != 0 + else None + ), + terminal=layer_id == config.num_hidden_layers - 1, + ) + + +class Qwen4ExpLayerExtensionMixin: def _init_qwen4_exp_layer_extensions( self, config: Qwen4ExpTextConfig, @@ -1423,17 +1481,16 @@ class Qwen4ExpLayerExtensionMixin: self.hc_count = config.hc_count self.hidden_size = config.hidden_size self.ple = None + self._ple_forward_batch = None + self._ple_batch = None - for attr_name in ( - "input_layernorm", - "post_attention_layernorm", - "attn_boundary", - "ffn_boundary", - ): + # The gated reads normalize the streams themselves, so these layers + # have neither of the norms their base class builds. + for attr_name in ("input_layernorm", "post_attention_layernorm"): if hasattr(self, attr_name): delattr(self, attr_name) - if (layer_id + 1) in config.ple_layer_ids: + if _has_ple(layer_id, config): ple_layer_ids_sorted = sorted(set(config.ple_layer_ids)) ple_layer_index = { abs_id: index for index, abs_id in enumerate(ple_layer_ids_sorted) @@ -1467,126 +1524,78 @@ class Qwen4ExpLayerExtensionMixin: use_mix=True, use_combine=True, ) + self.attn_boundary, self.ffn_boundary = _build_qwen4_exp_stages( + GatedResidualState( + expand=self._widen_streams, + attn_mix=self._attn_mix, + ffn_mix=self.mlp_hyper_connection.mix, + attn_combine=self.attn_hyper_connection.combine, + ffn_combine=self.mlp_hyper_connection.combine, + ).residual_ops(), + sparse=isinstance(self.mlp, Qwen2MoeSparseMoeBlock), + layer_id=layer_id, + config=config, + ) - def _prepare_qwen4_exp_attn( - self, - hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], - forward_batch: ForwardBatch, - *, - ple_batch: Optional[Union[_PLEBatch, _BreakablePLEBatch]], - ): + def _widen_streams(self, hidden_states: torch.Tensor) -> torch.Tensor: + """Open the layer stack's input into one stream per hyper connection. + An MTP draft is handed streams that are already widened.""" hc_dim = self.hc_count * self.hidden_size - if hidden_states.shape[-1] != hc_dim: - assert hidden_states.shape[-1] == self.hidden_size - hidden_states = torch.cat( - [hidden_states for _ in range(self.hc_count)], dim=-1 - ) - - if self.ple is not None: - if is_in_breakable_cuda_graph() and forward_batch.forward_mode.is_extend(): - ple_query = ( - hidden_states if residual is None else hidden_states + residual - ) - hidden_states = hidden_states + _breakable_ple_forward( - ple=self.ple, hidden_states=ple_query, batch=ple_batch - ) - elif ple_batch is None: - if not _get_ple_forward_mode(forward_batch).is_idle(): - raise RuntimeError( - "non-idle Qwen4 PLE forward is missing its batch" - ) - self.ple.forward_idle(forward_batch) - else: - ple_query = ( - hidden_states if residual is None else hidden_states + residual - ) - hidden_states = hidden_states + self.ple( - ple_query, forward_batch, ple_batch - ) - - hidden_states, residual = self.attn_hyper_connection.mix(hidden_states) - return hidden_states, residual - - def _prepare_qwen4_exp_mlp( - self, - hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], - forward_batch: ForwardBatch, - ): - if not forward_batch.forward_mode.is_idle(): - hidden_states = attn_tp_all_reduce(hidden_states) - hidden_states = self.attn_hyper_connection.combine(hidden_states, residual) - hidden_states, residual = self.mlp_hyper_connection.mix(hidden_states) - return hidden_states, residual - - def _qwen4_exp_use_dp_moe_gather(self) -> bool: - return get_parallel().attn_dp_size > 1 and get_moe_a2a_backend().is_none() - - def _qwen4_exp_use_attn_tp_a2a_scatter(self) -> bool: - return get_parallel().attn_tp_size > 1 and not get_moe_a2a_backend().is_none() - - def _run_qwen4_exp_mlp( - self, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - ) -> torch.Tensor: - if not self.config.num_experts: - return self.mlp(hidden_states) - - use_dp_moe_gather = self._qwen4_exp_use_dp_moe_gather() - use_attn_tp_a2a_scatter = self._qwen4_exp_use_attn_tp_a2a_scatter() - - if use_dp_moe_gather: - hidden_states, local_hidden_states = ( - get_global_dp_buffer(get_parallel().tp_group), - hidden_states, - ) - dp_gather_replicate(hidden_states, local_hidden_states, forward_batch) - elif hidden_states.shape[0] == 0 and get_moe_a2a_backend().is_none(): - # Only safe to short-circuit an empty batch when the MoE holds no collective; - # under deepep an idle DP rank must still join dispatch/combine or peers hang. + if hidden_states.shape[-1] == hc_dim: return hidden_states + assert hidden_states.shape[-1] == self.hidden_size + return torch.cat([hidden_states for _ in range(self.hc_count)], dim=-1) - attn_tp_chunks = None - if use_attn_tp_a2a_scatter: - attn_tp_size = get_parallel().attn_tp_size - attn_tp_chunks = list(hidden_states.tensor_split(attn_tp_size)) - hidden_states = attn_tp_chunks[get_parallel().attn_tp_rank].contiguous() + def _attn_mix(self, residual: torch.Tensor): + """The attention stage's read. A PLE layer's embedding is this layer's + own contribution to the streams and joins them before the read.""" + if self.ple is not None: + residual = self._add_ple_embedding(residual) + return self.attn_hyper_connection.mix(residual) - use_reduce_scatterv = use_dp_moe_gather and should_use_dp_reduce_scatterv() - with get_forward().scoped(mlp_reduce_scatter=use_reduce_scatterv): - hidden_states = self.mlp(hidden_states, forward_batch) - - if use_dp_moe_gather: - hidden_states, global_hidden_states = ( - get_local_dp_buffer(get_parallel().tp_group), - hidden_states, + def _add_ple_embedding(self, residual: torch.Tensor) -> torch.Tensor: + forward_batch, ple_batch = self._ple_forward_batch, self._ple_batch + if is_in_breakable_cuda_graph() and forward_batch.forward_mode.is_extend(): + return residual + _breakable_ple_forward( + ple=self.ple, hidden_states=residual, batch=ple_batch ) - if use_reduce_scatterv: - get_parallel().tp_group.reduce_scatterv( - global_hidden_states, - output=hidden_states, - sizes=get_dp_global_num_tokens(), - ) - else: - dp_scatter(hidden_states, global_hidden_states, forward_batch) - elif use_attn_tp_a2a_scatter: - assert attn_tp_chunks is not None - gathered = [torch.empty_like(t) for t in attn_tp_chunks] - attn_tp_all_gather(gathered, hidden_states.contiguous()) - hidden_states = torch.cat(gathered) + if ple_batch is None: + if not _get_ple_forward_mode(forward_batch).is_idle(): + raise RuntimeError("non-idle Qwen4 PLE forward is missing its batch") + self.ple.forward_idle(forward_batch) + return residual + return residual + self.ple(residual, forward_batch, ple_batch) - return hidden_states - - def _postprocess_qwen4_exp_layer( + def _prepare_attn_stage( self, hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], forward_batch: ForwardBatch, - ): - hidden_states = self.mlp_hyper_connection.combine(hidden_states, residual) - return hidden_states, None + ple_batch: Optional[Union[_PLEBatch, _BreakablePLEBatch]], + ) -> torch.Tensor: + # The read runs inside prepare and reaches the PLE batch through the + # layer for the duration of that call. + if self.ple is None: + return self.attn_boundary.prepare(hidden_states, forward_batch) + self._ple_forward_batch, self._ple_batch = forward_batch, ple_batch + try: + return self.attn_boundary.prepare(hidden_states, forward_batch) + finally: + self._ple_forward_batch = self._ple_batch = None + + def _run_ffn_stage( + self, hidden_states: torch.Tensor, forward_batch: ForwardBatch + ) -> torch.Tensor: + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + if isinstance(self.mlp, Qwen2MoeSparseMoeBlock): + hidden_states = self.mlp( + hidden_states, + forward_batch, + defer_finalize=ffn_exit.defer_moe_finalize, + ) + else: + hidden_states = self.mlp(hidden_states) + return ffn_exit.finish(hidden_states) class Qwen4ExpLinearDecoderLayer( @@ -1601,32 +1610,33 @@ class Qwen4ExpLinearDecoderLayer( alt_stream: Optional[torch.cuda.Stream] = None, is_nextn: bool = False, ) -> None: - super().__init__(config, layer_id, quant_config, prefix, alt_stream, is_nextn) + super().__init__( + config, + layer_id, + quant_config, + prefix, + alt_stream, + is_nextn, + build_stages=False, + ) self._init_qwen4_exp_layer_extensions(config, layer_id, quant_config, prefix) def forward( self, hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], **kwargs, ): forward_batch = kwargs.get("forward_batch", None) - hidden_states, residual = self._prepare_qwen4_exp_attn( - hidden_states, - residual, - forward_batch, - ple_batch=kwargs.get("ple_batch"), + hidden_states = self._prepare_attn_stage( + hidden_states, forward_batch, kwargs.get("ple_batch") ) if not forward_batch.forward_mode.is_idle(): hidden_states = self.linear_attn(hidden_states, forward_batch) - hidden_states, residual = self._prepare_qwen4_exp_mlp( - hidden_states, residual, forward_batch - ) - hidden_states = self._run_qwen4_exp_mlp(hidden_states, forward_batch) - return self._postprocess_qwen4_exp_layer(hidden_states, residual, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + return self._run_ffn_stage(hidden_states, forward_batch) class Qwen4ExpAttentionDecoderLayer( @@ -1642,7 +1652,15 @@ class Qwen4ExpAttentionDecoderLayer( is_nextn: bool = False, ) -> None: config.attn_output_gate = True - super().__init__(config, layer_id, quant_config, prefix, alt_stream, is_nextn) + super().__init__( + config, + layer_id, + quant_config, + prefix, + alt_stream, + is_nextn, + build_stages=False, + ) from sglang.srt.layers.attention.qsa.config import is_qwen_qsa from sglang.srt.layers.attention.qsa.glue import build_qsa_indexer @@ -1779,15 +1797,11 @@ class Qwen4ExpAttentionDecoderLayer( self, positions: torch.Tensor, hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], forward_batch: ForwardBatch, **kwargs: Any, ): - hidden_states, residual = self._prepare_qwen4_exp_attn( - hidden_states, - residual, - forward_batch, - ple_batch=kwargs.get("ple_batch"), + hidden_states = self._prepare_attn_stage( + hidden_states, forward_batch, kwargs.get("ple_batch") ) if not forward_batch.forward_mode.is_idle(): @@ -1797,11 +1811,8 @@ class Qwen4ExpAttentionDecoderLayer( forward_batch=forward_batch, ) - hidden_states, residual = self._prepare_qwen4_exp_mlp( - hidden_states, residual, forward_batch - ) - hidden_states = self._run_qwen4_exp_mlp(hidden_states, forward_batch) - return self._postprocess_qwen4_exp_layer(hidden_states, residual, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + return self._run_ffn_stage(hidden_states, forward_batch) ALL_DECODER_LAYER_TYPES = { @@ -1857,6 +1868,8 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM): if self.pp_group.is_last_rank else PPMissingLayer() ) + # The stack's terminal read mixes the streams down; there is no final norm. + self._mix_streams = _MixStreams(self.hyper_connection_mixer) def forward( self, @@ -1871,18 +1884,17 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM): hidden_states = inputs_embeds else: hidden_states = self.embed_tokens(input_ids) - residual = None + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] # Only the first stage embeds multimodal inputs; the last stage needs # them for the MTP draft prefill, so relay them with the hidden states. mm_input_embeds = pp_proxy_tensors.tensors.get("mm_input_embeds") if mm_input_embeds is not None: forward_batch.mm_input_embeds = mm_input_embeds - # The hyper-connection streams ride in the widened hidden state; - # there is no separate residual at a PP boundary (hc_hidden_size contract). - residual = None + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( + pp_proxy_tensors, forward_batch + ) breakable_ple = ( self.has_ple @@ -1916,10 +1928,9 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM): batch=ple_batch, forward_batch=forward_batch ) with get_global_expert_distribution_recorder().with_current_layer(i): - hidden_states, residual = layer( + hidden_states = layer( positions=positions, hidden_states=hidden_states, - residual=residual, forward_batch=forward_batch, ple_batch=ple_batch, captured_last_layer_outputs=( @@ -1935,13 +1946,17 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM): _commit_ple_batch(batch=ple_batch, forward_batch=forward_batch) if not self.pp_group.is_last_rank: - proxy_tensors = {"hidden_states": hidden_states} + # The streams ride in the widened hidden state: the FFN write-back + # already wrote them, so to_pp sends no separate residual. + proxy_tensors = residual_batch.to_pp(hidden_states, forward_batch) if forward_batch.mm_input_embeds is not None: proxy_tensors["mm_input_embeds"] = forward_batch.mm_input_embeds - return PPProxyTensors(proxy_tensors) + return proxy_tensors hc_hidden_states = hidden_states - hidden_states, _ = self.hyper_connection_mixer.mix(hidden_states) + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, self._mix_streams + ) if not forward_batch.forward_mode.is_idle(): return hidden_states, hc_hidden_states diff --git a/test/registered/unit/layer_boundary/test_gated_residual_ops.py b/test/registered/unit/layer_boundary/test_gated_residual_ops.py new file mode 100644 index 0000000000..bcef5cb9a6 --- /dev/null +++ b/test/registered/unit/layer_boundary/test_gated_residual_ops.py @@ -0,0 +1,256 @@ +import unittest +from unittest.mock import patch + +import torch + +from sglang.srt.layers.layer_boundary.residual.gated import GatedResidualState +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +HC_COUNT = 3 +HIDDEN = 4 +WIDE = HC_COUNT * HIDDEN + + +def _expand(hidden_states): + """The layer stack's input widened into streams; already widened passes + through, as the write-back returns streams.""" + if hidden_states.shape[-1] == WIDE: + return hidden_states + return torch.cat([hidden_states for _ in range(HC_COUNT)], dim=-1) + + +def _normalize(residual, seed): + """A stand-in for the per-branch norm each stage's mix applies.""" + return residual * (0.5 + seed) + + +def _mix(seed): + """Stands in for GatedResidual.mix: returns the mixed input and the pair + the write-back consumes, the streams and their normalization.""" + + def mix(hyper_input): + if hyper_input.shape[0] == 0: + empty = hyper_input.new_empty((0, HIDDEN)) + # An empty batch puts the un-normalized streams in the second slot. + return empty, (hyper_input, hyper_input) + normed = _normalize(hyper_input, seed) + mixed = normed.unflatten(-1, (HC_COUNT, HIDDEN)).mean(dim=-2) + return mixed, (hyper_input, normed) + + return mix + + +def _combine(block_output, residuals): + """Stands in for GatedResidual.combine: the injection coefficient is + computed at write time from the normalized residual the read produced.""" + hyper_input, normed = residuals + if block_output.shape[0] == 0: + return hyper_input + coefficient = 2 * torch.sigmoid( + normed.unflatten(-1, (HC_COUNT, HIDDEN)).mean(dim=-1) + ) + injected = block_output.unsqueeze(-2) * coefficient.unsqueeze(-1) + return (hyper_input.unflatten(-1, (HC_COUNT, HIDDEN)) + injected).flatten(-2) + + +def _state(attn_mix=None): + return GatedResidualState( + expand=_expand, + attn_mix=attn_mix if attn_mix is not None else _mix(1), + ffn_mix=_mix(2), + attn_combine=_combine, + ffn_combine=_combine, + ) + + +class TestGatedResidualOps(CustomTestCase): + def setUp(self): + self.hidden = torch.arange(2 * HIDDEN, dtype=torch.float32).reshape(2, HIDDEN) + self.output = torch.full((2, HIDDEN), 5.0) + + def test_entry_widens_the_stack_input(self): + ops = _state().residual_ops() + residual = ops.attn_readout.init_residual(self.hidden) + self.assertEqual(residual.shape, (2, WIDE)) + for stream in range(HC_COUNT): + torch.testing.assert_close( + residual[:, stream * HIDDEN : (stream + 1) * HIDDEN], self.hidden + ) + + def test_entry_passes_already_widened_streams_through(self): + """An MTP draft is handed streams that are already widened.""" + ops = _state().residual_ops() + widened = torch.ones(2, WIDE) + self.assertIs(ops.attn_readout.init_residual(widened), widened) + + def test_the_write_back_uses_the_residual_its_own_read_normalized(self): + state = _state() + ops = state.residual_ops() + residual = ops.attn_readout.init_residual(self.hidden) + + self.assertIsNone(state.normed) + _, residual = ops.attn_readout.read(residual, None) + normed = state.normed + self.assertIsNotNone(normed) + torch.testing.assert_close(normed, _normalize(residual, 1)) + + torch.testing.assert_close( + ops.attn_update.update(self.output, residual), + _combine(self.output, (residual, normed)), + ) + + def test_the_write_back_reads_the_carried_value_not_a_fresh_one(self): + """The coefficient is computed at write time, so what crosses from the + read is the normalized residual itself. Replacing it must move the + write-back; recomputing from the streams would not.""" + state = _state() + ops = state.residual_ops() + residual = ops.attn_readout.init_residual(self.hidden) + _, residual = ops.attn_readout.read(residual, None) + + state.normed = torch.full_like(state.normed, -3.0) + torch.testing.assert_close( + ops.attn_update.update(self.output, residual), + _combine(self.output, (residual, state.normed)), + ) + + def test_ffn_write_back_clears_the_carried_value(self): + state = _state() + ops = state.residual_ops() + residual = ops.attn_readout.init_residual(self.hidden) + ops.attn_readout.read(residual, None) + _, residual = ops.ffn_readout.update_and_read( + ops.attn_update, self.output, residual, None + ) + self.assertIsNotNone(state.normed) + + ops.ffn_update.update(torch.full((2, HIDDEN), 7.0), residual) + # Nothing survives the layer: the next layer's read produces its own. + self.assertIsNone(state.normed) + + def test_the_ffn_input_is_read_from_the_updated_streams(self): + state = _state() + ops = state.residual_ops() + residual = ops.attn_readout.init_residual(self.hidden) + _, residual = ops.attn_readout.read(residual, None) + attn_normed = state.normed + + got_input, got_residual = ops.ffn_readout.update_and_read( + ops.attn_update, self.output, residual, None + ) + want_residual = _combine(self.output, (residual, attn_normed)) + want_input, (_, want_normed) = _mix(2)(want_residual) + torch.testing.assert_close(got_residual, want_residual) + torch.testing.assert_close(got_input, want_input) + torch.testing.assert_close(state.normed, want_normed) + + def test_a_read_may_contribute_to_the_streams_before_mixing(self): + """A layer whose own contribution joins the streams ahead of its read + (the Qwen4 PLE embedding) folds it into the mix it hands the state, so + the read returns streams this layer has added to.""" + contribution = torch.full((2, WIDE), 0.25) + state = _state(attn_mix=lambda residual: _mix(1)(residual + contribution)) + ops = state.residual_ops() + entered = ops.attn_readout.init_residual(self.hidden) + + _, residual = ops.attn_readout.read(entered, None) + torch.testing.assert_close(residual, entered + contribution) + # The write-back lands on the contributed streams, once. + torch.testing.assert_close( + ops.attn_update.update(self.output, residual), + _combine(self.output, (entered + contribution, state.normed)), + ) + + def test_an_empty_batch_keeps_the_streams_and_their_width(self): + state = _state() + ops = state.residual_ops() + residual = ops.attn_readout.init_residual(torch.empty(0, HIDDEN)) + self.assertEqual(residual.shape, (0, WIDE)) + + hidden_states, residual = ops.attn_readout.read(residual, None) + self.assertEqual(hidden_states.shape, (0, HIDDEN)) + # The mix leaves the un-normalized streams behind on an empty batch. + torch.testing.assert_close(state.normed, residual) + + written = ops.attn_update.update(torch.empty(0, HIDDEN), residual) + torch.testing.assert_close(written, residual) + + def test_the_terminal_write_back_leaves_the_streams_widened(self): + """The stack's terminal read mixes the streams down, so the FFN + write-back must not contract them.""" + ops = _state().residual_ops() + residual = ops.attn_readout.init_residual(self.hidden) + ops.attn_readout.read(residual, None) + _, residual = ops.ffn_readout.update_and_read( + ops.attn_update, self.output, residual, None + ) + written = ops.ffn_update.update(torch.full((2, HIDDEN), 7.0), residual) + self.assertEqual(written.shape, (2, WIDE)) + + def test_declared_capabilities(self): + ops = _state().residual_ops() + # A gated injection is not an add, so the sum it writes into must be + # complete first and no add+norm fusion may claim it. + self.assertFalse(ops.attn_update.is_plain_add) + self.assertFalse(ops.ffn_update.is_plain_add) + # Each read normalizes the streams itself. + self.assertFalse(ops.attn_readout.is_plain_norm) + self.assertFalse(ops.ffn_readout.is_plain_norm) + # The FFN writes at its exit; the attention leaves its write-back to + # the FFN input's read, inside the same layer. + self.assertTrue(ops.ffn_update.applied_at_exit) + self.assertFalse(ops.attn_update.applied_at_exit) + self.assertFalse(ops.attn_update.outlives_layer) + self.assertFalse(ops.ffn_update.outlives_layer) + + def test_rejects_reads_it_does_not_implement(self): + ops = _state().residual_ops() + residual = ops.attn_readout.init_residual(self.hidden) + with self.assertRaises(NotImplementedError): + ops.attn_readout.read(residual, None, quant_format="fp8") + # The mix normalizes the streams, so a separate norm has no place. + with self.assertRaises(NotImplementedError): + ops.attn_readout.read(residual, lambda hidden_states: hidden_states) + with self.assertRaises(NotImplementedError): + ops.attn_readout.update_and_read( + ops.ffn_update, self.hidden, residual, None + ) + with self.assertRaises(NotImplementedError): + ops.ffn_readout.read(residual, None) + + def test_ffn_read_rejects_another_layers_update(self): + ops = _state().residual_ops() + other = _state().residual_ops() + residual = ops.attn_readout.init_residual(self.hidden) + ops.attn_readout.read(residual, None) + with self.assertRaises(NotImplementedError): + ops.ffn_readout.update_and_read( + other.attn_update, self.hidden, residual, None + ) + + def test_attn_tp_slice_moves_the_carried_value_with_the_streams(self): + state = _state() + ops = state.residual_ops() + residual = torch.arange(4 * WIDE, dtype=torch.float32).reshape(4, WIDE) + state.normed = _normalize(residual, 1) + with patch( + "sglang.srt.layers.layer_boundary.residual.gated.get_parallel" + ) as parallel: + parallel.return_value.attn_tp_rank = 1 + parallel.return_value.attn_tp_size = 2 + sliced = ops.attn_update.slice_residual_attn_tp(residual) + torch.testing.assert_close(sliced, residual[2:]) + # The normalized rows must follow the stream rows they scale. + torch.testing.assert_close(state.normed, _normalize(residual, 1)[2:]) + + def test_attn_tp_gather_is_rejected(self): + ops = _state().residual_ops() + with self.assertRaises(NotImplementedError): + ops.attn_update.gather_residual_attn_tp(self.hidden) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layer_boundary/test_moe_output_reduction.py b/test/registered/unit/layer_boundary/test_moe_output_reduction.py index 196f7b5be8..a189008c37 100644 --- a/test/registered/unit/layer_boundary/test_moe_output_reduction.py +++ b/test/registered/unit/layer_boundary/test_moe_output_reduction.py @@ -164,96 +164,82 @@ class TestReplicatedMoeOutput(CustomTestCase): class TestModelsWithExplicitDpCompletion(CustomTestCase): def test_direct_dp_exits_publish_their_selected_sum(self): # Execute the real orchestration methods without constructing weights. - # These models own their DP exit instead of using StageBoundary. + # DeepSeek V4 owns its DP exit instead of using a stage boundary. import __future__ - for filename, method in ( - ("deepseek_v4.py", "_run_moe_ffn_dp_sync"), - ("qwen4_exp.py", "_run_qwen4_exp_mlp"), - ): - path = MODELS_DIR / filename - node = next( - n - for n in ast.walk(ast.parse(path.read_text())) - if isinstance(n, ast.FunctionDef) and n.name == method - ) - for use_rsv in (False, True): - with self.subTest(model=filename, reduce_scatterv=use_rsv): - trace = [] + filename, method = "deepseek_v4.py", "_run_moe_ffn_dp_sync" + path = MODELS_DIR / filename + node = next( + n + for n in ast.walk(ast.parse(path.read_text())) + if isinstance(n, ast.FunctionDef) and n.name == method + ) + for use_rsv in (False, True): + with self.subTest(model=filename, reduce_scatterv=use_rsv): + trace = [] - def rsv(value, *, output, sizes): - trace.append("RSv") - output.copy_(value[:2] * 2) + def rsv(value, *, output, sizes): + trace.append("RSv") + output.copy_(value[:2] * 2) - def scatter(output, value, batch): - trace.append("slice") - output.copy_(value[:2]) + def scatter(output, value, batch): + trace.append("slice") + output.copy_(value[:2]) - group = types.SimpleNamespace(reduce_scatterv=rsv) - parallel = types.SimpleNamespace( - attn_dp_size=2, attn_tp_size=1, tp_size=2, tp_group=group - ) - namespace = dict( - torch=torch, - get_forward=get_forward, - get_parallel=lambda: parallel, - get_moe_a2a_backend=lambda: a2a(), - should_use_dp_reduce_scatterv=lambda: use_rsv, - is_dp_gatherv_active=lambda: False, - is_cp_active=lambda batch: False, - envs=types.SimpleNamespace( - SGLANG_DP_USE_REDUCE_SCATTER=types.SimpleNamespace( - get=lambda: False - ) - ), - _SHARED_EXPERT_LOCAL=False, - nullcontext=contextlib.nullcontext, - get_global_dp_buffer=lambda g: torch.empty(4, 3), - get_local_dp_buffer=lambda g: torch.empty(2, 3), - get_dp_global_num_tokens=lambda: [2, 2], - dp_gather_replicate=lambda output, value, batch: output.fill_( - 1 - ), - dp_scatter=scatter, - ) - exec( - compile( - ast.Module(body=[node], type_ignores=[]), - str(path), - "exec", - flags=__future__.annotations.compiler_flag, - ), - namespace, - ) - model = types.SimpleNamespace( - dsa_enable_prefill_cp=False, - config=types.SimpleNamespace(num_experts=4), - _qwen4_exp_use_dp_moe_gather=lambda: True, - _qwen4_exp_use_attn_tp_a2a_scatter=lambda: False, - mlp=lambda value, batch, **kwargs: reduce_moe_output(value), - ) - batch = types.SimpleNamespace( - dp_padding_mode=types.SimpleNamespace(is_max_len=lambda: True) - ) - kwargs = ( - dict(input_ids=None, input_ids_global=None) - if filename == "deepseek_v4.py" - else {} - ) - with ( - moe_config(), - get_forward().scoped(mlp_reduce_scatter=False), - patch( - "sglang.srt.distributed.communication_op.tensor_model_parallel_all_reduce", - side_effect=lambda value: trace.append("AR") or value * 2, - ), - ): - output = namespace[method]( - model, torch.ones(2, 3), batch, **kwargs + group = types.SimpleNamespace(reduce_scatterv=rsv) + parallel = types.SimpleNamespace( + attn_dp_size=2, attn_tp_size=1, tp_size=2, tp_group=group + ) + namespace = dict( + torch=torch, + get_forward=get_forward, + get_parallel=lambda: parallel, + get_moe_a2a_backend=lambda: a2a(), + should_use_dp_reduce_scatterv=lambda: use_rsv, + is_dp_gatherv_active=lambda: False, + is_cp_active=lambda batch: False, + envs=types.SimpleNamespace( + SGLANG_DP_USE_REDUCE_SCATTER=types.SimpleNamespace( + get=lambda: False ) - self.assertFalse(get_forward().mlp_reduce_scatter) - self.assertEqual(trace, ["RSv"] if use_rsv else ["AR", "slice"]) - torch.testing.assert_close(output, torch.full((2, 3), 2.0)) + ), + _SHARED_EXPERT_LOCAL=False, + nullcontext=contextlib.nullcontext, + get_global_dp_buffer=lambda g: torch.empty(4, 3), + get_local_dp_buffer=lambda g: torch.empty(2, 3), + get_dp_global_num_tokens=lambda: [2, 2], + dp_gather_replicate=lambda output, value, batch: output.fill_(1), + dp_scatter=scatter, + ) + exec( + compile( + ast.Module(body=[node], type_ignores=[]), + str(path), + "exec", + flags=__future__.annotations.compiler_flag, + ), + namespace, + ) + model = types.SimpleNamespace( + dsa_enable_prefill_cp=False, + mlp=lambda value, batch, **kwargs: reduce_moe_output(value), + ) + batch = types.SimpleNamespace( + dp_padding_mode=types.SimpleNamespace(is_max_len=lambda: True) + ) + kwargs = dict(input_ids=None, input_ids_global=None) + with ( + moe_config(), + get_forward().scoped(mlp_reduce_scatter=False), + patch( + "sglang.srt.distributed.communication_op.tensor_model_parallel_all_reduce", + side_effect=lambda value: trace.append("AR") or value * 2, + ), + ): + output = namespace[method](model, torch.ones(2, 3), batch, **kwargs) + self.assertFalse(get_forward().mlp_reduce_scatter) + self.assertEqual(trace, ["RSv"] if use_rsv else ["AR", "slice"]) + torch.testing.assert_close(output, torch.full((2, 3), 2.0)) if __name__ == "__main__": diff --git a/test/registered/unit/layer_boundary/test_qwen4_exp_ple_rows.py b/test/registered/unit/layer_boundary/test_qwen4_exp_ple_rows.py new file mode 100644 index 0000000000..7ed01177e1 --- /dev/null +++ b/test/registered/unit/layer_boundary/test_qwen4_exp_ple_rows.py @@ -0,0 +1,64 @@ +"""A Qwen4-exp PLE layer reads its input on full attention rows.""" + +import unittest +from types import SimpleNamespace + +import test_declared_decoder_boundary as fixture + +from sglang.srt.layers.layer_boundary.contracts import BatchVariant +from sglang.srt.layers.layer_boundary.layout import TokenAxis +from sglang.srt.layers.layer_boundary.residual.gated import GatedResidualState +from sglang.srt.models.qwen4_exp import _build_qwen4_exp_stages +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _residual_ops(): + def unused(*args, **kwargs): + raise AssertionError("construction only") + + return GatedResidualState( + expand=unused, + attn_mix=unused, + ffn_mix=unused, + attn_combine=unused, + ffn_combine=unused, + ).residual_ops() + + +def _sliced_over_attention_tp(layout): + return TokenAxis.ATTN_TP in layout.sharded + + +class TestQwen4ExpPleRows(CustomTestCase): + def test_the_ffn_before_a_ple_layer_hands_on_full_rows(self): + # An all-to-all MoE with attention TP keeps the residual on this + # rank's slice of the rows between layers; the PLE embedding is + # computed for every row, so its layer must read the full rows. + config = SimpleNamespace(num_hidden_layers=4, ple_layer_ids=[3]) + with fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=2), a2a=True): + layers = [ + _build_qwen4_exp_stages( + _residual_ops(), sparse=True, layer_id=layer_id, config=config + ) + for layer_id in range(config.num_hidden_layers) + ] + ordinary = BatchVariant.ORDINARY + ffn_hands_on = [ + _sliced_over_attention_tp(ffn.plan.edges[ordinary].outgoing.residual_to) + for _, ffn in layers + ] + attn_reads = [ + _sliced_over_attention_tp(attn.plan.edges[ordinary].incoming.residual) + for attn, _ in layers + ] + # Layer 2 has the PLE: layer 1's FFN gathers the rows back, the last + # layer's FFN ends the stack on attention rows anyway. + self.assertEqual(ffn_hands_on, [True, False, True, False]) + self.assertEqual(attn_reads, [False, True, False, True]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py b/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py index 20239e7c4b..2891822432 100644 --- a/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py +++ b/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py @@ -33,6 +33,20 @@ _BUILDS_STAGES = ( ) +# A vocabulary-parallel lookup leaves each rank holding only its own shard's +# rows, so the class completes that sum itself. An embedding is never a decoder +# stage, so no sum inside one is a stage output and the whole class is exempt. +# Only a class that names one of these bases directly is exempt. +_VOCAB_PARALLEL_BASES = ("VocabParallelEmbedding", "ParallelLMHead") + + +def _is_vocab_parallel(node) -> bool: + return isinstance(node, ast.ClassDef) and any( + base in _VOCAB_PARALLEL_BASES + for base in (ast.unparse(parent) for parent in node.bases) + ) + + def _call_name(node: ast.Call): func = node.func return func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", None) @@ -40,10 +54,13 @@ def _call_name(node: ast.Call): def _unguarded_sums(source: str): """Sum calls not under ``if ... self.reduce_results ...``: such a branch is - a shared class built without stage boundaries by another model.""" + a shared class built without stage boundaries by another model. Sums inside + a vocabulary-parallel embedding complete its lookup, not a stage output.""" found = [] def visit(node, guarded): + if _is_vocab_parallel(node): + return if isinstance(node, ast.If) and "reduce_results" in ast.unparse(node.test): for child in node.body: visit(child, True) diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index 33113359a7..b7642b81c1 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -242,6 +242,8 @@ class TestBoundaryReductionDefaults(CustomTestCase): ("Qwen3ForCausalLM", "ar"), ("Qwen3Model", "ar"), ("MossVLForConditionalGeneration", "ar"), + ("Qwen4ExpForConditionalGeneration", "ar"), + ("Qwen4ExpForCausalLMMTP", "ar"), ("BailingMoELinearForCausalLM", "rsv"), ("BailingMoeV2_5ForCausalLM", "rsv"), ("LongcatFlashForCausalLM", "rsv"),