[Refactor] Build the Qwen4 experimental decoders from stage boundaries (#42478)

This commit is contained in:
Cheng Wan
2026-10-04 16:38:36 -07:00
committed by GitHub
parent 92d60351e2
commit f07c1a398e
10 changed files with 854 additions and 304 deletions
@@ -18,6 +18,8 @@ _AR_ARCHITECTURES = {
"Qwen3ForRewardModel",
"MossVLForCausalLM",
"MossVLForConditionalGeneration",
"Qwen4ExpForConditionalGeneration",
"Qwen4ExpForCausalLMMTP",
}
@@ -72,6 +72,9 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import (
PLAIN_ADD,
PLAIN_RESIDUAL_OPS,
)
from sglang.srt.layers.layer_boundary.residual.gated import (
GatedResidualState,
)
from sglang.srt.layers.layer_boundary.residual.ihc import (
IHCState,
)
@@ -96,6 +99,7 @@ __all__ = [
"FfnExit",
"FfnInputFusion",
"DeferredFinalize",
"GatedResidualState",
"IHCState",
"LayerResidualOps",
"Layout",
@@ -0,0 +1,198 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Gated-injection hyper-connection residual streams."""
from dataclasses import dataclass
from typing import Callable, Optional
import torch
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
from sglang.srt.runtime_context import get_parallel
@dataclass
class GatedResidualState:
"""A layer's residual as hc_count streams flattened into one tensor: a
stage reads its input by normalizing the streams and mixing them down with
per-stream gates, and writes its output back scaled by an injection
coefficient computed from that same normalized residual.
Unlike the hyper-connection streams in mhc.py and ihc.py, whose read
produces the coefficient its write-back consumes, the coefficient here is
computed at write time. What has to survive from a stage's read to its
write-back is therefore the normalized residual itself, which this state
holds. Parameters belong to the owning layer's modules.
Args:
expand: Widens the layer stack's input into the stream representation,
and passes an already widened residual through.
attn_mix: The attention stage's read. A layer whose own contribution
joins the streams before that read (the Qwen4 PLE embedding) folds
it in here, so this state does not need to know about it.
ffn_mix: The FFN stage's read.
attn_combine: Writes the attention output back into the streams.
ffn_combine: Writes the FFN output back into the streams.
"""
expand: Callable
attn_mix: Callable
ffn_mix: Callable
attn_combine: Callable
ffn_combine: Callable
# Produced by a stage's read and consumed by that same stage's write-back.
normed: Optional[torch.Tensor] = None
def _read(self, mix, residual, out_norm):
if out_norm is not None:
raise NotImplementedError(
"a gated hyper-connection read with a separate norm; the read "
"normalizes the streams itself"
)
hidden_states, (residual, self.normed) = mix(residual)
return hidden_states, residual
def read_attn_input(self, residual, out_norm=None):
return self._read(self.attn_mix, residual, out_norm)
def update_and_read_ffn_input(self, hidden_states, residual, out_norm=None):
residual = self.apply_attn_combine(hidden_states, residual)
return self._read(self.ffn_mix, residual, out_norm)
def apply_attn_combine(self, hidden_states, residual):
return self.attn_combine(hidden_states, (residual, self.normed))
def apply_ffn_combine(self, hidden_states, residual):
return self.ffn_combine(hidden_states, (residual, self.normed))
def clear_coefficients(self):
self.normed = None
def slice_residual_attn_tp(self, residual):
parallel = get_parallel()
rank, size = parallel.attn_tp_rank, parallel.attn_tp_size
self.normed = self.normed.tensor_split(size)[rank]
return residual.tensor_split(size)[rank]
def gather_residual_attn_tp(self, residual):
raise NotImplementedError(
"Unsupported: gated hyper-connection normed-residual allgather is "
"not implemented."
)
def residual_ops(self) -> LayerResidualOps:
return LayerResidualOps(
attn_readout=_AttnReadout(self),
attn_update=_AttnUpdate(self),
ffn_readout=_FfnReadout(self),
ffn_update=_FfnUpdate(self),
)
class _AttnReadout:
"""The normalized, gated mix of the streams, from streams that already hold
the previous layer's output: such a layer takes its input written back."""
is_plain_norm = False
reads_before_dp_gather = False
def __init__(self, state: GatedResidualState):
self.state = state
def init_residual(self, hidden_states):
return self.state.expand(hidden_states)
def read(self, residual, norm, quant_format="", post_residual_addition=None):
if quant_format:
raise NotImplementedError(
f"a gated hyper-connection attention input in {quant_format=}"
)
return self.state.read_attn_input(residual, out_norm=norm)
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
raise NotImplementedError(
"a gated hyper-connection layer takes its input written back"
)
class _AttnUpdate:
"""The gated injection of the attention output, using the normalized
residual its read produced. The injection is not a plain add, so it runs
only once the sum it writes in is complete."""
is_plain_add = False
applied_at_exit = False
outlives_layer = False
def __init__(self, state: GatedResidualState):
self.state = state
def update(self, hidden_states, residual):
return self.state.apply_attn_combine(hidden_states, residual)
def slice_residual_attn_tp(self, residual):
return self.state.slice_residual_attn_tp(residual)
def gather_residual_attn_tp(self, residual):
return self.state.gather_residual_attn_tp(residual)
class _FfnReadout:
"""The attention output's injection and the FFN input's mix."""
is_plain_norm = False
reads_before_dp_gather = False
def __init__(self, state: GatedResidualState):
self.state = state
def init_residual(self, hidden_states):
return self.state.expand(hidden_states)
def read(self, residual, norm, quant_format="", post_residual_addition=None):
raise NotImplementedError(
"a gated hyper-connection FFN input read without its attention"
)
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
if not isinstance(update, _AttnUpdate) or update.state is not self.state:
raise NotImplementedError(
f"a gated hyper-connection FFN input after {update=}"
)
return self.state.update_and_read_ffn_input(
hidden_states, residual, out_norm=norm
)
class _FfnUpdate:
"""The gated injection of the FFN output, which this layer runs itself. The
streams stay widened: the layer stack's terminal read mixes them down."""
is_plain_add = False
applied_at_exit = True
outlives_layer = False
def __init__(self, state: GatedResidualState):
self.state = state
def update(self, hidden_states, residual):
hidden_states = self.state.apply_ffn_combine(hidden_states, residual)
self.state.clear_coefficients()
return hidden_states
def slice_residual_attn_tp(self, residual):
return self.state.slice_residual_attn_tp(residual)
def gather_residual_attn_tp(self, residual):
return self.state.gather_residual_attn_tp(residual)
+64 -58
View File
@@ -1041,9 +1041,6 @@ class Qwen3_5GatedDeltaNet(nn.Module):
class Qwen3_5LinearDecoderLayer(nn.Module):
"""Qwen3.5 Decoder Layer with Linear Attention (GatedDeltaNet)."""
# True in a subclass built without stage boundaries.
_ffn_sums_itself = False
def __init__(
self,
config: Qwen3_5TextConfig,
@@ -1052,6 +1049,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
prefix: str = "",
alt_stream: Optional[torch.cuda.Stream] = None,
is_nextn: bool = False,
build_stages: bool = True,
) -> None:
super().__init__()
self.config = config
@@ -1076,7 +1074,8 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")),
is_nextn=is_nextn,
support_shared_expert_fusion=not _disable_shared_experts_fusion(),
reduce_results=self._ffn_sums_itself,
# The stage boundary completes this stage's sum.
reduce_results=False,
)
is_layer_sparse = True
is_previous_layer_sparse = True
@@ -1088,7 +1087,8 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")),
reduce_results=self._ffn_sums_itself,
# The stage boundary completes this stage's sum.
reduce_results=False,
)
_maybe_enable_silu_fp4_quant_fusion(self.mlp)
is_layer_sparse = False
@@ -1106,33 +1106,37 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
# it. Otherwise, stay on the plain AR+RMSNorm path.
accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
boundary_fusions = _layer_fusions(config, is_nextn)
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(
read=NormQuantReadout(
fp8_input=Fp8Input.TUPLE_AND_BF16 if accepts_fp8_input else None
)
# A subclass that brings its own residual builds the stages itself.
if build_stages:
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(
read=NormQuantReadout(
fp8_input=Fp8Input.TUPLE_AND_BF16
if accepts_fp8_input
else None
)
),
self.input_layernorm,
{
"fusions": boundary_fusions,
},
),
self.input_layernorm,
{
"fusions": boundary_fusions,
},
),
(
declare_ffn(
sparse=is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
(
declare_ffn(
sparse=is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
{"fusions": boundary_fusions},
),
self.post_attention_layernorm,
{"fusions": boundary_fusions},
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
self,
@@ -1174,9 +1178,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
class Qwen3_5AttentionDecoderLayer(nn.Module):
"""Qwen3.5 Decoder Layer with Full Attention."""
# See Qwen3_5LinearDecoderLayer._ffn_sums_itself.
_ffn_sums_itself = False
def __init__(
self,
config: Qwen3_5TextConfig,
@@ -1185,6 +1186,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
prefix: str = "",
alt_stream: Optional[torch.cuda.Stream] = None,
is_nextn: bool = False,
build_stages: bool = True,
) -> None:
super().__init__()
self.config = config
@@ -1279,7 +1281,8 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix.replace(".self_attn", "")),
reduce_results=self._ffn_sums_itself,
# The stage boundary completes this stage's sum.
reduce_results=False,
)
is_layer_sparse = False
is_previous_layer_sparse = False
@@ -1297,7 +1300,8 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix.replace(".self_attn", "")),
is_nextn=is_nextn,
support_shared_expert_fusion=not _disable_shared_experts_fusion(),
reduce_results=self._ffn_sums_itself,
# The stage boundary completes this stage's sum.
reduce_results=False,
)
is_layer_sparse = True
is_previous_layer_sparse = True
@@ -1317,33 +1321,35 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
# when qkv_proj can consume the returned quantized tuple.
accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj)
boundary_fusions = _layer_fusions(config, is_nextn)
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(
read=NormQuantReadout(
fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None
)
# A subclass that brings its own residual builds the stages itself.
if build_stages:
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(
read=NormQuantReadout(
fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None
)
),
self.input_layernorm,
{
"fusions": boundary_fusions,
},
),
self.input_layernorm,
{
"fusions": boundary_fusions,
},
),
(
declare_ffn(
sparse=is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
(
declare_ffn(
sparse=is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
{"fusions": boundary_fusions},
),
self.post_attention_layernorm,
{"fusions": boundary_fusions},
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
self.alt_stream = alt_stream
+176 -161
View File
@@ -26,13 +26,9 @@ from sglang.srt.layers.attention.linear.utils import (
)
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather,
attn_tp_all_reduce,
dp_gather_replicate,
dp_scatter,
get_dp_global_num_tokens,
get_global_dp_buffer,
get_local_dp_buffer,
is_allocation_symmetric,
is_dp_attention_enabled,
)
@@ -40,10 +36,17 @@ from sglang.srt.layers.hyperconnection import (
GatedResidual,
HyperConnectionConfig,
)
from sglang.srt.layers.layer_boundary import get_attn_tp_context
from sglang.srt.layers.layer_boundary import (
ExitRows,
GatedResidualState,
declare_attn,
declare_ffn,
get_attn_tp_context,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.layers.moe import get_moe_a2a_backend, should_use_dp_reduce_scatterv
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.modelopt_quant import (
@@ -70,6 +73,7 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo
get_tc_piecewise_forward_context,
)
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.qwen2_moe import Qwen2MoeSparseMoeBlock
from sglang.srt.models.qwen3_5 import (
Qwen3_5AttentionDecoderLayer,
Qwen3_5ForCausalLM,
@@ -82,7 +86,7 @@ from sglang.srt.models.qwen4_exp_ple_table import (
make_ple_file_prefetcher,
make_ple_file_rss_trimmer,
)
from sglang.srt.runtime_context import get_forward, get_parallel
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_bool_env_var, is_hip, logger
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
@@ -1409,10 +1413,64 @@ class Qwen4ExpPLELayer(nn.Module):
return _pad_token_rows(output, batch.physical_tokens)
class Qwen4ExpLayerExtensionMixin:
# These layers drop the stage boundaries they inherit.
_ffn_sums_itself = True
class _MixStreams:
"""The layer stack's terminal read: the gated mix down to one stream. The
stack ends on the mix, with no final norm after it."""
def __init__(self, hyper_connection_mixer):
self.hyper_connection_mixer = hyper_connection_mixer
def __call__(self, hidden_states):
return self.hyper_connection_mixer.mix(hidden_states)[0]
def _has_ple(layer_id, config) -> bool:
return (layer_id + 1) in config.ple_layer_ids
def _ffn_exit_rows(layer_id, config):
"""A PLE layer's read adds an embedding computed for every row, so the FFN
before it hands its output on as full attention rows."""
return ExitRows.ATTENTION if _has_ple(layer_id + 1, config) else None
def _build_qwen4_exp_stages(residual, *, sparse, layer_id, config):
"""The attention and FFN stage boundaries of one gated hyper-connection layer.
Each read normalizes the streams itself, so neither stage binds a norm, and
the writes are gated injections, so neither stage offers a fused
add-and-norm candidate.
"""
return make_stages(
(
declare_attn(read=residual.attn_readout, update=residual.attn_update),
None,
),
(
declare_ffn(
sparse=sparse,
next_layer_sparse=sparse,
read=residual.ffn_readout,
update=residual.ffn_update,
exit_rows=_ffn_exit_rows(layer_id, config),
),
None,
),
previous=(
declare_ffn(
sparse=sparse,
next_layer_sparse=sparse,
update=residual.ffn_update,
exit_rows=_ffn_exit_rows(layer_id - 1, config),
)
if layer_id != 0
else None
),
terminal=layer_id == config.num_hidden_layers - 1,
)
class Qwen4ExpLayerExtensionMixin:
def _init_qwen4_exp_layer_extensions(
self,
config: Qwen4ExpTextConfig,
@@ -1423,17 +1481,16 @@ class Qwen4ExpLayerExtensionMixin:
self.hc_count = config.hc_count
self.hidden_size = config.hidden_size
self.ple = None
self._ple_forward_batch = None
self._ple_batch = None
for attr_name in (
"input_layernorm",
"post_attention_layernorm",
"attn_boundary",
"ffn_boundary",
):
# The gated reads normalize the streams themselves, so these layers
# have neither of the norms their base class builds.
for attr_name in ("input_layernorm", "post_attention_layernorm"):
if hasattr(self, attr_name):
delattr(self, attr_name)
if (layer_id + 1) in config.ple_layer_ids:
if _has_ple(layer_id, config):
ple_layer_ids_sorted = sorted(set(config.ple_layer_ids))
ple_layer_index = {
abs_id: index for index, abs_id in enumerate(ple_layer_ids_sorted)
@@ -1467,126 +1524,78 @@ class Qwen4ExpLayerExtensionMixin:
use_mix=True,
use_combine=True,
)
self.attn_boundary, self.ffn_boundary = _build_qwen4_exp_stages(
GatedResidualState(
expand=self._widen_streams,
attn_mix=self._attn_mix,
ffn_mix=self.mlp_hyper_connection.mix,
attn_combine=self.attn_hyper_connection.combine,
ffn_combine=self.mlp_hyper_connection.combine,
).residual_ops(),
sparse=isinstance(self.mlp, Qwen2MoeSparseMoeBlock),
layer_id=layer_id,
config=config,
)
def _prepare_qwen4_exp_attn(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
*,
ple_batch: Optional[Union[_PLEBatch, _BreakablePLEBatch]],
):
def _widen_streams(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Open the layer stack's input into one stream per hyper connection.
An MTP draft is handed streams that are already widened."""
hc_dim = self.hc_count * self.hidden_size
if hidden_states.shape[-1] != hc_dim:
assert hidden_states.shape[-1] == self.hidden_size
hidden_states = torch.cat(
[hidden_states for _ in range(self.hc_count)], dim=-1
)
if self.ple is not None:
if is_in_breakable_cuda_graph() and forward_batch.forward_mode.is_extend():
ple_query = (
hidden_states if residual is None else hidden_states + residual
)
hidden_states = hidden_states + _breakable_ple_forward(
ple=self.ple, hidden_states=ple_query, batch=ple_batch
)
elif ple_batch is None:
if not _get_ple_forward_mode(forward_batch).is_idle():
raise RuntimeError(
"non-idle Qwen4 PLE forward is missing its batch"
)
self.ple.forward_idle(forward_batch)
else:
ple_query = (
hidden_states if residual is None else hidden_states + residual
)
hidden_states = hidden_states + self.ple(
ple_query, forward_batch, ple_batch
)
hidden_states, residual = self.attn_hyper_connection.mix(hidden_states)
return hidden_states, residual
def _prepare_qwen4_exp_mlp(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
):
if not forward_batch.forward_mode.is_idle():
hidden_states = attn_tp_all_reduce(hidden_states)
hidden_states = self.attn_hyper_connection.combine(hidden_states, residual)
hidden_states, residual = self.mlp_hyper_connection.mix(hidden_states)
return hidden_states, residual
def _qwen4_exp_use_dp_moe_gather(self) -> bool:
return get_parallel().attn_dp_size > 1 and get_moe_a2a_backend().is_none()
def _qwen4_exp_use_attn_tp_a2a_scatter(self) -> bool:
return get_parallel().attn_tp_size > 1 and not get_moe_a2a_backend().is_none()
def _run_qwen4_exp_mlp(
self,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> torch.Tensor:
if not self.config.num_experts:
return self.mlp(hidden_states)
use_dp_moe_gather = self._qwen4_exp_use_dp_moe_gather()
use_attn_tp_a2a_scatter = self._qwen4_exp_use_attn_tp_a2a_scatter()
if use_dp_moe_gather:
hidden_states, local_hidden_states = (
get_global_dp_buffer(get_parallel().tp_group),
hidden_states,
)
dp_gather_replicate(hidden_states, local_hidden_states, forward_batch)
elif hidden_states.shape[0] == 0 and get_moe_a2a_backend().is_none():
# Only safe to short-circuit an empty batch when the MoE holds no collective;
# under deepep an idle DP rank must still join dispatch/combine or peers hang.
if hidden_states.shape[-1] == hc_dim:
return hidden_states
assert hidden_states.shape[-1] == self.hidden_size
return torch.cat([hidden_states for _ in range(self.hc_count)], dim=-1)
attn_tp_chunks = None
if use_attn_tp_a2a_scatter:
attn_tp_size = get_parallel().attn_tp_size
attn_tp_chunks = list(hidden_states.tensor_split(attn_tp_size))
hidden_states = attn_tp_chunks[get_parallel().attn_tp_rank].contiguous()
def _attn_mix(self, residual: torch.Tensor):
"""The attention stage's read. A PLE layer's embedding is this layer's
own contribution to the streams and joins them before the read."""
if self.ple is not None:
residual = self._add_ple_embedding(residual)
return self.attn_hyper_connection.mix(residual)
use_reduce_scatterv = use_dp_moe_gather and should_use_dp_reduce_scatterv()
with get_forward().scoped(mlp_reduce_scatter=use_reduce_scatterv):
hidden_states = self.mlp(hidden_states, forward_batch)
if use_dp_moe_gather:
hidden_states, global_hidden_states = (
get_local_dp_buffer(get_parallel().tp_group),
hidden_states,
def _add_ple_embedding(self, residual: torch.Tensor) -> torch.Tensor:
forward_batch, ple_batch = self._ple_forward_batch, self._ple_batch
if is_in_breakable_cuda_graph() and forward_batch.forward_mode.is_extend():
return residual + _breakable_ple_forward(
ple=self.ple, hidden_states=residual, batch=ple_batch
)
if use_reduce_scatterv:
get_parallel().tp_group.reduce_scatterv(
global_hidden_states,
output=hidden_states,
sizes=get_dp_global_num_tokens(),
)
else:
dp_scatter(hidden_states, global_hidden_states, forward_batch)
elif use_attn_tp_a2a_scatter:
assert attn_tp_chunks is not None
gathered = [torch.empty_like(t) for t in attn_tp_chunks]
attn_tp_all_gather(gathered, hidden_states.contiguous())
hidden_states = torch.cat(gathered)
if ple_batch is None:
if not _get_ple_forward_mode(forward_batch).is_idle():
raise RuntimeError("non-idle Qwen4 PLE forward is missing its batch")
self.ple.forward_idle(forward_batch)
return residual
return residual + self.ple(residual, forward_batch, ple_batch)
return hidden_states
def _postprocess_qwen4_exp_layer(
def _prepare_attn_stage(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
):
hidden_states = self.mlp_hyper_connection.combine(hidden_states, residual)
return hidden_states, None
ple_batch: Optional[Union[_PLEBatch, _BreakablePLEBatch]],
) -> torch.Tensor:
# The read runs inside prepare and reaches the PLE batch through the
# layer for the duration of that call.
if self.ple is None:
return self.attn_boundary.prepare(hidden_states, forward_batch)
self._ple_forward_batch, self._ple_batch = forward_batch, ple_batch
try:
return self.attn_boundary.prepare(hidden_states, forward_batch)
finally:
self._ple_forward_batch = self._ple_batch = None
def _run_ffn_stage(
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
) -> torch.Tensor:
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if isinstance(self.mlp, Qwen2MoeSparseMoeBlock):
hidden_states = self.mlp(
hidden_states,
forward_batch,
defer_finalize=ffn_exit.defer_moe_finalize,
)
else:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
class Qwen4ExpLinearDecoderLayer(
@@ -1601,32 +1610,33 @@ class Qwen4ExpLinearDecoderLayer(
alt_stream: Optional[torch.cuda.Stream] = None,
is_nextn: bool = False,
) -> None:
super().__init__(config, layer_id, quant_config, prefix, alt_stream, is_nextn)
super().__init__(
config,
layer_id,
quant_config,
prefix,
alt_stream,
is_nextn,
build_stages=False,
)
self._init_qwen4_exp_layer_extensions(config, layer_id, quant_config, prefix)
def forward(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
**kwargs,
):
forward_batch = kwargs.get("forward_batch", None)
hidden_states, residual = self._prepare_qwen4_exp_attn(
hidden_states,
residual,
forward_batch,
ple_batch=kwargs.get("ple_batch"),
hidden_states = self._prepare_attn_stage(
hidden_states, forward_batch, kwargs.get("ple_batch")
)
if not forward_batch.forward_mode.is_idle():
hidden_states = self.linear_attn(hidden_states, forward_batch)
hidden_states, residual = self._prepare_qwen4_exp_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self._run_qwen4_exp_mlp(hidden_states, forward_batch)
return self._postprocess_qwen4_exp_layer(hidden_states, residual, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
return self._run_ffn_stage(hidden_states, forward_batch)
class Qwen4ExpAttentionDecoderLayer(
@@ -1642,7 +1652,15 @@ class Qwen4ExpAttentionDecoderLayer(
is_nextn: bool = False,
) -> None:
config.attn_output_gate = True
super().__init__(config, layer_id, quant_config, prefix, alt_stream, is_nextn)
super().__init__(
config,
layer_id,
quant_config,
prefix,
alt_stream,
is_nextn,
build_stages=False,
)
from sglang.srt.layers.attention.qsa.config import is_qwen_qsa
from sglang.srt.layers.attention.qsa.glue import build_qsa_indexer
@@ -1779,15 +1797,11 @@ class Qwen4ExpAttentionDecoderLayer(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
**kwargs: Any,
):
hidden_states, residual = self._prepare_qwen4_exp_attn(
hidden_states,
residual,
forward_batch,
ple_batch=kwargs.get("ple_batch"),
hidden_states = self._prepare_attn_stage(
hidden_states, forward_batch, kwargs.get("ple_batch")
)
if not forward_batch.forward_mode.is_idle():
@@ -1797,11 +1811,8 @@ class Qwen4ExpAttentionDecoderLayer(
forward_batch=forward_batch,
)
hidden_states, residual = self._prepare_qwen4_exp_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self._run_qwen4_exp_mlp(hidden_states, forward_batch)
return self._postprocess_qwen4_exp_layer(hidden_states, residual, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
return self._run_ffn_stage(hidden_states, forward_batch)
ALL_DECODER_LAYER_TYPES = {
@@ -1857,6 +1868,8 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
if self.pp_group.is_last_rank
else PPMissingLayer()
)
# The stack's terminal read mixes the streams down; there is no final norm.
self._mix_streams = _MixStreams(self.hyper_connection_mixer)
def forward(
self,
@@ -1871,18 +1884,17 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
hidden_states = inputs_embeds
else:
hidden_states = self.embed_tokens(input_ids)
residual = None
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
# Only the first stage embeds multimodal inputs; the last stage needs
# them for the MTP draft prefill, so relay them with the hidden states.
mm_input_embeds = pp_proxy_tensors.tensors.get("mm_input_embeds")
if mm_input_embeds is not None:
forward_batch.mm_input_embeds = mm_input_embeds
# The hyper-connection streams ride in the widened hidden state;
# there is no separate residual at a PP boundary (hc_hidden_size contract).
residual = None
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
breakable_ple = (
self.has_ple
@@ -1916,10 +1928,9 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
batch=ple_batch, forward_batch=forward_batch
)
with get_global_expert_distribution_recorder().with_current_layer(i):
hidden_states, residual = layer(
hidden_states = layer(
positions=positions,
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
ple_batch=ple_batch,
captured_last_layer_outputs=(
@@ -1935,13 +1946,17 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
_commit_ple_batch(batch=ple_batch, forward_batch=forward_batch)
if not self.pp_group.is_last_rank:
proxy_tensors = {"hidden_states": hidden_states}
# The streams ride in the widened hidden state: the FFN write-back
# already wrote them, so to_pp sends no separate residual.
proxy_tensors = residual_batch.to_pp(hidden_states, forward_batch)
if forward_batch.mm_input_embeds is not None:
proxy_tensors["mm_input_embeds"] = forward_batch.mm_input_embeds
return PPProxyTensors(proxy_tensors)
return proxy_tensors
hc_hidden_states = hidden_states
hidden_states, _ = self.hyper_connection_mixer.mix(hidden_states)
hidden_states = residual_batch.final_norm(
hidden_states, forward_batch, self._mix_streams
)
if not forward_batch.forward_mode.is_idle():
return hidden_states, hc_hidden_states