[Refactor] Build the Qwen4 experimental decoders from stage boundaries (#42478)
This commit is contained in:
@@ -18,6 +18,8 @@ _AR_ARCHITECTURES = {
|
||||
"Qwen3ForRewardModel",
|
||||
"MossVLForCausalLM",
|
||||
"MossVLForConditionalGeneration",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
"Qwen4ExpForCausalLMMTP",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -72,6 +72,9 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
PLAIN_ADD,
|
||||
PLAIN_RESIDUAL_OPS,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.gated import (
|
||||
GatedResidualState,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.ihc import (
|
||||
IHCState,
|
||||
)
|
||||
@@ -96,6 +99,7 @@ __all__ = [
|
||||
"FfnExit",
|
||||
"FfnInputFusion",
|
||||
"DeferredFinalize",
|
||||
"GatedResidualState",
|
||||
"IHCState",
|
||||
"LayerResidualOps",
|
||||
"Layout",
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Gated-injection hyper-connection residual streams."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
@dataclass
|
||||
class GatedResidualState:
|
||||
"""A layer's residual as hc_count streams flattened into one tensor: a
|
||||
stage reads its input by normalizing the streams and mixing them down with
|
||||
per-stream gates, and writes its output back scaled by an injection
|
||||
coefficient computed from that same normalized residual.
|
||||
|
||||
Unlike the hyper-connection streams in mhc.py and ihc.py, whose read
|
||||
produces the coefficient its write-back consumes, the coefficient here is
|
||||
computed at write time. What has to survive from a stage's read to its
|
||||
write-back is therefore the normalized residual itself, which this state
|
||||
holds. Parameters belong to the owning layer's modules.
|
||||
|
||||
Args:
|
||||
expand: Widens the layer stack's input into the stream representation,
|
||||
and passes an already widened residual through.
|
||||
attn_mix: The attention stage's read. A layer whose own contribution
|
||||
joins the streams before that read (the Qwen4 PLE embedding) folds
|
||||
it in here, so this state does not need to know about it.
|
||||
ffn_mix: The FFN stage's read.
|
||||
attn_combine: Writes the attention output back into the streams.
|
||||
ffn_combine: Writes the FFN output back into the streams.
|
||||
"""
|
||||
|
||||
expand: Callable
|
||||
attn_mix: Callable
|
||||
ffn_mix: Callable
|
||||
attn_combine: Callable
|
||||
ffn_combine: Callable
|
||||
# Produced by a stage's read and consumed by that same stage's write-back.
|
||||
normed: Optional[torch.Tensor] = None
|
||||
|
||||
def _read(self, mix, residual, out_norm):
|
||||
if out_norm is not None:
|
||||
raise NotImplementedError(
|
||||
"a gated hyper-connection read with a separate norm; the read "
|
||||
"normalizes the streams itself"
|
||||
)
|
||||
hidden_states, (residual, self.normed) = mix(residual)
|
||||
return hidden_states, residual
|
||||
|
||||
def read_attn_input(self, residual, out_norm=None):
|
||||
return self._read(self.attn_mix, residual, out_norm)
|
||||
|
||||
def update_and_read_ffn_input(self, hidden_states, residual, out_norm=None):
|
||||
residual = self.apply_attn_combine(hidden_states, residual)
|
||||
return self._read(self.ffn_mix, residual, out_norm)
|
||||
|
||||
def apply_attn_combine(self, hidden_states, residual):
|
||||
return self.attn_combine(hidden_states, (residual, self.normed))
|
||||
|
||||
def apply_ffn_combine(self, hidden_states, residual):
|
||||
return self.ffn_combine(hidden_states, (residual, self.normed))
|
||||
|
||||
def clear_coefficients(self):
|
||||
self.normed = None
|
||||
|
||||
def slice_residual_attn_tp(self, residual):
|
||||
parallel = get_parallel()
|
||||
rank, size = parallel.attn_tp_rank, parallel.attn_tp_size
|
||||
self.normed = self.normed.tensor_split(size)[rank]
|
||||
return residual.tensor_split(size)[rank]
|
||||
|
||||
def gather_residual_attn_tp(self, residual):
|
||||
raise NotImplementedError(
|
||||
"Unsupported: gated hyper-connection normed-residual allgather is "
|
||||
"not implemented."
|
||||
)
|
||||
|
||||
def residual_ops(self) -> LayerResidualOps:
|
||||
return LayerResidualOps(
|
||||
attn_readout=_AttnReadout(self),
|
||||
attn_update=_AttnUpdate(self),
|
||||
ffn_readout=_FfnReadout(self),
|
||||
ffn_update=_FfnUpdate(self),
|
||||
)
|
||||
|
||||
|
||||
class _AttnReadout:
|
||||
"""The normalized, gated mix of the streams, from streams that already hold
|
||||
the previous layer's output: such a layer takes its input written back."""
|
||||
|
||||
is_plain_norm = False
|
||||
reads_before_dp_gather = False
|
||||
|
||||
def __init__(self, state: GatedResidualState):
|
||||
self.state = state
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
return self.state.expand(hidden_states)
|
||||
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
if quant_format:
|
||||
raise NotImplementedError(
|
||||
f"a gated hyper-connection attention input in {quant_format=}"
|
||||
)
|
||||
return self.state.read_attn_input(residual, out_norm=norm)
|
||||
|
||||
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
|
||||
raise NotImplementedError(
|
||||
"a gated hyper-connection layer takes its input written back"
|
||||
)
|
||||
|
||||
|
||||
class _AttnUpdate:
|
||||
"""The gated injection of the attention output, using the normalized
|
||||
residual its read produced. The injection is not a plain add, so it runs
|
||||
only once the sum it writes in is complete."""
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
outlives_layer = False
|
||||
|
||||
def __init__(self, state: GatedResidualState):
|
||||
self.state = state
|
||||
|
||||
def update(self, hidden_states, residual):
|
||||
return self.state.apply_attn_combine(hidden_states, residual)
|
||||
|
||||
def slice_residual_attn_tp(self, residual):
|
||||
return self.state.slice_residual_attn_tp(residual)
|
||||
|
||||
def gather_residual_attn_tp(self, residual):
|
||||
return self.state.gather_residual_attn_tp(residual)
|
||||
|
||||
|
||||
class _FfnReadout:
|
||||
"""The attention output's injection and the FFN input's mix."""
|
||||
|
||||
is_plain_norm = False
|
||||
reads_before_dp_gather = False
|
||||
|
||||
def __init__(self, state: GatedResidualState):
|
||||
self.state = state
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
return self.state.expand(hidden_states)
|
||||
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
raise NotImplementedError(
|
||||
"a gated hyper-connection FFN input read without its attention"
|
||||
)
|
||||
|
||||
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
|
||||
if not isinstance(update, _AttnUpdate) or update.state is not self.state:
|
||||
raise NotImplementedError(
|
||||
f"a gated hyper-connection FFN input after {update=}"
|
||||
)
|
||||
return self.state.update_and_read_ffn_input(
|
||||
hidden_states, residual, out_norm=norm
|
||||
)
|
||||
|
||||
|
||||
class _FfnUpdate:
|
||||
"""The gated injection of the FFN output, which this layer runs itself. The
|
||||
streams stay widened: the layer stack's terminal read mixes them down."""
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
outlives_layer = False
|
||||
|
||||
def __init__(self, state: GatedResidualState):
|
||||
self.state = state
|
||||
|
||||
def update(self, hidden_states, residual):
|
||||
hidden_states = self.state.apply_ffn_combine(hidden_states, residual)
|
||||
self.state.clear_coefficients()
|
||||
return hidden_states
|
||||
|
||||
def slice_residual_attn_tp(self, residual):
|
||||
return self.state.slice_residual_attn_tp(residual)
|
||||
|
||||
def gather_residual_attn_tp(self, residual):
|
||||
return self.state.gather_residual_attn_tp(residual)
|
||||
@@ -1041,9 +1041,6 @@ class Qwen3_5GatedDeltaNet(nn.Module):
|
||||
class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
"""Qwen3.5 Decoder Layer with Linear Attention (GatedDeltaNet)."""
|
||||
|
||||
# True in a subclass built without stage boundaries.
|
||||
_ffn_sums_itself = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen3_5TextConfig,
|
||||
@@ -1052,6 +1049,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
prefix: str = "",
|
||||
alt_stream: Optional[torch.cuda.Stream] = None,
|
||||
is_nextn: bool = False,
|
||||
build_stages: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -1076,7 +1074,8 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")),
|
||||
is_nextn=is_nextn,
|
||||
support_shared_expert_fusion=not _disable_shared_experts_fusion(),
|
||||
reduce_results=self._ffn_sums_itself,
|
||||
# The stage boundary completes this stage's sum.
|
||||
reduce_results=False,
|
||||
)
|
||||
is_layer_sparse = True
|
||||
is_previous_layer_sparse = True
|
||||
@@ -1088,7 +1087,8 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")),
|
||||
reduce_results=self._ffn_sums_itself,
|
||||
# The stage boundary completes this stage's sum.
|
||||
reduce_results=False,
|
||||
)
|
||||
_maybe_enable_silu_fp4_quant_fusion(self.mlp)
|
||||
is_layer_sparse = False
|
||||
@@ -1106,33 +1106,37 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
# it. Otherwise, stay on the plain AR+RMSNorm path.
|
||||
accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
|
||||
boundary_fusions = _layer_fusions(config, is_nextn)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=NormQuantReadout(
|
||||
fp8_input=Fp8Input.TUPLE_AND_BF16 if accepts_fp8_input else None
|
||||
)
|
||||
# A subclass that brings its own residual builds the stages itself.
|
||||
if build_stages:
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=NormQuantReadout(
|
||||
fp8_input=Fp8Input.TUPLE_AND_BF16
|
||||
if accepts_fp8_input
|
||||
else None
|
||||
)
|
||||
),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"fusions": boundary_fusions,
|
||||
},
|
||||
),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"fusions": boundary_fusions,
|
||||
},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
{"fusions": boundary_fusions},
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
{"fusions": boundary_fusions},
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -1174,9 +1178,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
"""Qwen3.5 Decoder Layer with Full Attention."""
|
||||
|
||||
# See Qwen3_5LinearDecoderLayer._ffn_sums_itself.
|
||||
_ffn_sums_itself = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen3_5TextConfig,
|
||||
@@ -1185,6 +1186,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
prefix: str = "",
|
||||
alt_stream: Optional[torch.cuda.Stream] = None,
|
||||
is_nextn: bool = False,
|
||||
build_stages: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -1279,7 +1281,8 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix.replace(".self_attn", "")),
|
||||
reduce_results=self._ffn_sums_itself,
|
||||
# The stage boundary completes this stage's sum.
|
||||
reduce_results=False,
|
||||
)
|
||||
is_layer_sparse = False
|
||||
is_previous_layer_sparse = False
|
||||
@@ -1297,7 +1300,8 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix.replace(".self_attn", "")),
|
||||
is_nextn=is_nextn,
|
||||
support_shared_expert_fusion=not _disable_shared_experts_fusion(),
|
||||
reduce_results=self._ffn_sums_itself,
|
||||
# The stage boundary completes this stage's sum.
|
||||
reduce_results=False,
|
||||
)
|
||||
is_layer_sparse = True
|
||||
is_previous_layer_sparse = True
|
||||
@@ -1317,33 +1321,35 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
# when qkv_proj can consume the returned quantized tuple.
|
||||
accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj)
|
||||
boundary_fusions = _layer_fusions(config, is_nextn)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=NormQuantReadout(
|
||||
fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None
|
||||
)
|
||||
# A subclass that brings its own residual builds the stages itself.
|
||||
if build_stages:
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=NormQuantReadout(
|
||||
fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None
|
||||
)
|
||||
),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"fusions": boundary_fusions,
|
||||
},
|
||||
),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"fusions": boundary_fusions,
|
||||
},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
{"fusions": boundary_fusions},
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
{"fusions": boundary_fusions},
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
self.alt_stream = alt_stream
|
||||
|
||||
|
||||
@@ -26,13 +26,9 @@ from sglang.srt.layers.attention.linear.utils import (
|
||||
)
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather,
|
||||
attn_tp_all_reduce,
|
||||
dp_gather_replicate,
|
||||
dp_scatter,
|
||||
get_dp_global_num_tokens,
|
||||
get_global_dp_buffer,
|
||||
get_local_dp_buffer,
|
||||
is_allocation_symmetric,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -40,10 +36,17 @@ from sglang.srt.layers.hyperconnection import (
|
||||
GatedResidual,
|
||||
HyperConnectionConfig,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
ExitRows,
|
||||
GatedResidualState,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
get_attn_tp_context,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend, should_use_dp_reduce_scatterv
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.modelopt_quant import (
|
||||
@@ -70,6 +73,7 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo
|
||||
get_tc_piecewise_forward_context,
|
||||
)
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.qwen2_moe import Qwen2MoeSparseMoeBlock
|
||||
from sglang.srt.models.qwen3_5 import (
|
||||
Qwen3_5AttentionDecoderLayer,
|
||||
Qwen3_5ForCausalLM,
|
||||
@@ -82,7 +86,7 @@ from sglang.srt.models.qwen4_exp_ple_table import (
|
||||
make_ple_file_prefetcher,
|
||||
make_ple_file_rss_trimmer,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_forward, get_parallel
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip, logger
|
||||
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
|
||||
@@ -1409,10 +1413,64 @@ class Qwen4ExpPLELayer(nn.Module):
|
||||
return _pad_token_rows(output, batch.physical_tokens)
|
||||
|
||||
|
||||
class Qwen4ExpLayerExtensionMixin:
|
||||
# These layers drop the stage boundaries they inherit.
|
||||
_ffn_sums_itself = True
|
||||
class _MixStreams:
|
||||
"""The layer stack's terminal read: the gated mix down to one stream. The
|
||||
stack ends on the mix, with no final norm after it."""
|
||||
|
||||
def __init__(self, hyper_connection_mixer):
|
||||
self.hyper_connection_mixer = hyper_connection_mixer
|
||||
|
||||
def __call__(self, hidden_states):
|
||||
return self.hyper_connection_mixer.mix(hidden_states)[0]
|
||||
|
||||
|
||||
def _has_ple(layer_id, config) -> bool:
|
||||
return (layer_id + 1) in config.ple_layer_ids
|
||||
|
||||
|
||||
def _ffn_exit_rows(layer_id, config):
|
||||
"""A PLE layer's read adds an embedding computed for every row, so the FFN
|
||||
before it hands its output on as full attention rows."""
|
||||
return ExitRows.ATTENTION if _has_ple(layer_id + 1, config) else None
|
||||
|
||||
|
||||
def _build_qwen4_exp_stages(residual, *, sparse, layer_id, config):
|
||||
"""The attention and FFN stage boundaries of one gated hyper-connection layer.
|
||||
|
||||
Each read normalizes the streams itself, so neither stage binds a norm, and
|
||||
the writes are gated injections, so neither stage offers a fused
|
||||
add-and-norm candidate.
|
||||
"""
|
||||
return make_stages(
|
||||
(
|
||||
declare_attn(read=residual.attn_readout, update=residual.attn_update),
|
||||
None,
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=sparse,
|
||||
next_layer_sparse=sparse,
|
||||
read=residual.ffn_readout,
|
||||
update=residual.ffn_update,
|
||||
exit_rows=_ffn_exit_rows(layer_id, config),
|
||||
),
|
||||
None,
|
||||
),
|
||||
previous=(
|
||||
declare_ffn(
|
||||
sparse=sparse,
|
||||
next_layer_sparse=sparse,
|
||||
update=residual.ffn_update,
|
||||
exit_rows=_ffn_exit_rows(layer_id - 1, config),
|
||||
)
|
||||
if layer_id != 0
|
||||
else None
|
||||
),
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
|
||||
class Qwen4ExpLayerExtensionMixin:
|
||||
def _init_qwen4_exp_layer_extensions(
|
||||
self,
|
||||
config: Qwen4ExpTextConfig,
|
||||
@@ -1423,17 +1481,16 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
self.hc_count = config.hc_count
|
||||
self.hidden_size = config.hidden_size
|
||||
self.ple = None
|
||||
self._ple_forward_batch = None
|
||||
self._ple_batch = None
|
||||
|
||||
for attr_name in (
|
||||
"input_layernorm",
|
||||
"post_attention_layernorm",
|
||||
"attn_boundary",
|
||||
"ffn_boundary",
|
||||
):
|
||||
# The gated reads normalize the streams themselves, so these layers
|
||||
# have neither of the norms their base class builds.
|
||||
for attr_name in ("input_layernorm", "post_attention_layernorm"):
|
||||
if hasattr(self, attr_name):
|
||||
delattr(self, attr_name)
|
||||
|
||||
if (layer_id + 1) in config.ple_layer_ids:
|
||||
if _has_ple(layer_id, config):
|
||||
ple_layer_ids_sorted = sorted(set(config.ple_layer_ids))
|
||||
ple_layer_index = {
|
||||
abs_id: index for index, abs_id in enumerate(ple_layer_ids_sorted)
|
||||
@@ -1467,126 +1524,78 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
use_mix=True,
|
||||
use_combine=True,
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = _build_qwen4_exp_stages(
|
||||
GatedResidualState(
|
||||
expand=self._widen_streams,
|
||||
attn_mix=self._attn_mix,
|
||||
ffn_mix=self.mlp_hyper_connection.mix,
|
||||
attn_combine=self.attn_hyper_connection.combine,
|
||||
ffn_combine=self.mlp_hyper_connection.combine,
|
||||
).residual_ops(),
|
||||
sparse=isinstance(self.mlp, Qwen2MoeSparseMoeBlock),
|
||||
layer_id=layer_id,
|
||||
config=config,
|
||||
)
|
||||
|
||||
def _prepare_qwen4_exp_attn(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
ple_batch: Optional[Union[_PLEBatch, _BreakablePLEBatch]],
|
||||
):
|
||||
def _widen_streams(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Open the layer stack's input into one stream per hyper connection.
|
||||
An MTP draft is handed streams that are already widened."""
|
||||
hc_dim = self.hc_count * self.hidden_size
|
||||
if hidden_states.shape[-1] != hc_dim:
|
||||
assert hidden_states.shape[-1] == self.hidden_size
|
||||
hidden_states = torch.cat(
|
||||
[hidden_states for _ in range(self.hc_count)], dim=-1
|
||||
)
|
||||
|
||||
if self.ple is not None:
|
||||
if is_in_breakable_cuda_graph() and forward_batch.forward_mode.is_extend():
|
||||
ple_query = (
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
)
|
||||
hidden_states = hidden_states + _breakable_ple_forward(
|
||||
ple=self.ple, hidden_states=ple_query, batch=ple_batch
|
||||
)
|
||||
elif ple_batch is None:
|
||||
if not _get_ple_forward_mode(forward_batch).is_idle():
|
||||
raise RuntimeError(
|
||||
"non-idle Qwen4 PLE forward is missing its batch"
|
||||
)
|
||||
self.ple.forward_idle(forward_batch)
|
||||
else:
|
||||
ple_query = (
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
)
|
||||
hidden_states = hidden_states + self.ple(
|
||||
ple_query, forward_batch, ple_batch
|
||||
)
|
||||
|
||||
hidden_states, residual = self.attn_hyper_connection.mix(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
def _prepare_qwen4_exp_mlp(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = attn_tp_all_reduce(hidden_states)
|
||||
hidden_states = self.attn_hyper_connection.combine(hidden_states, residual)
|
||||
hidden_states, residual = self.mlp_hyper_connection.mix(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
def _qwen4_exp_use_dp_moe_gather(self) -> bool:
|
||||
return get_parallel().attn_dp_size > 1 and get_moe_a2a_backend().is_none()
|
||||
|
||||
def _qwen4_exp_use_attn_tp_a2a_scatter(self) -> bool:
|
||||
return get_parallel().attn_tp_size > 1 and not get_moe_a2a_backend().is_none()
|
||||
|
||||
def _run_qwen4_exp_mlp(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
if not self.config.num_experts:
|
||||
return self.mlp(hidden_states)
|
||||
|
||||
use_dp_moe_gather = self._qwen4_exp_use_dp_moe_gather()
|
||||
use_attn_tp_a2a_scatter = self._qwen4_exp_use_attn_tp_a2a_scatter()
|
||||
|
||||
if use_dp_moe_gather:
|
||||
hidden_states, local_hidden_states = (
|
||||
get_global_dp_buffer(get_parallel().tp_group),
|
||||
hidden_states,
|
||||
)
|
||||
dp_gather_replicate(hidden_states, local_hidden_states, forward_batch)
|
||||
elif hidden_states.shape[0] == 0 and get_moe_a2a_backend().is_none():
|
||||
# Only safe to short-circuit an empty batch when the MoE holds no collective;
|
||||
# under deepep an idle DP rank must still join dispatch/combine or peers hang.
|
||||
if hidden_states.shape[-1] == hc_dim:
|
||||
return hidden_states
|
||||
assert hidden_states.shape[-1] == self.hidden_size
|
||||
return torch.cat([hidden_states for _ in range(self.hc_count)], dim=-1)
|
||||
|
||||
attn_tp_chunks = None
|
||||
if use_attn_tp_a2a_scatter:
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
attn_tp_chunks = list(hidden_states.tensor_split(attn_tp_size))
|
||||
hidden_states = attn_tp_chunks[get_parallel().attn_tp_rank].contiguous()
|
||||
def _attn_mix(self, residual: torch.Tensor):
|
||||
"""The attention stage's read. A PLE layer's embedding is this layer's
|
||||
own contribution to the streams and joins them before the read."""
|
||||
if self.ple is not None:
|
||||
residual = self._add_ple_embedding(residual)
|
||||
return self.attn_hyper_connection.mix(residual)
|
||||
|
||||
use_reduce_scatterv = use_dp_moe_gather and should_use_dp_reduce_scatterv()
|
||||
with get_forward().scoped(mlp_reduce_scatter=use_reduce_scatterv):
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
|
||||
if use_dp_moe_gather:
|
||||
hidden_states, global_hidden_states = (
|
||||
get_local_dp_buffer(get_parallel().tp_group),
|
||||
hidden_states,
|
||||
def _add_ple_embedding(self, residual: torch.Tensor) -> torch.Tensor:
|
||||
forward_batch, ple_batch = self._ple_forward_batch, self._ple_batch
|
||||
if is_in_breakable_cuda_graph() and forward_batch.forward_mode.is_extend():
|
||||
return residual + _breakable_ple_forward(
|
||||
ple=self.ple, hidden_states=residual, batch=ple_batch
|
||||
)
|
||||
if use_reduce_scatterv:
|
||||
get_parallel().tp_group.reduce_scatterv(
|
||||
global_hidden_states,
|
||||
output=hidden_states,
|
||||
sizes=get_dp_global_num_tokens(),
|
||||
)
|
||||
else:
|
||||
dp_scatter(hidden_states, global_hidden_states, forward_batch)
|
||||
elif use_attn_tp_a2a_scatter:
|
||||
assert attn_tp_chunks is not None
|
||||
gathered = [torch.empty_like(t) for t in attn_tp_chunks]
|
||||
attn_tp_all_gather(gathered, hidden_states.contiguous())
|
||||
hidden_states = torch.cat(gathered)
|
||||
if ple_batch is None:
|
||||
if not _get_ple_forward_mode(forward_batch).is_idle():
|
||||
raise RuntimeError("non-idle Qwen4 PLE forward is missing its batch")
|
||||
self.ple.forward_idle(forward_batch)
|
||||
return residual
|
||||
return residual + self.ple(residual, forward_batch, ple_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
def _postprocess_qwen4_exp_layer(
|
||||
def _prepare_attn_stage(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
hidden_states = self.mlp_hyper_connection.combine(hidden_states, residual)
|
||||
return hidden_states, None
|
||||
ple_batch: Optional[Union[_PLEBatch, _BreakablePLEBatch]],
|
||||
) -> torch.Tensor:
|
||||
# The read runs inside prepare and reaches the PLE batch through the
|
||||
# layer for the duration of that call.
|
||||
if self.ple is None:
|
||||
return self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
self._ple_forward_batch, self._ple_batch = forward_batch, ple_batch
|
||||
try:
|
||||
return self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
finally:
|
||||
self._ple_forward_batch = self._ple_batch = None
|
||||
|
||||
def _run_ffn_stage(
|
||||
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if isinstance(self.mlp, Qwen2MoeSparseMoeBlock):
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
defer_finalize=ffn_exit.defer_moe_finalize,
|
||||
)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
class Qwen4ExpLinearDecoderLayer(
|
||||
@@ -1601,32 +1610,33 @@ class Qwen4ExpLinearDecoderLayer(
|
||||
alt_stream: Optional[torch.cuda.Stream] = None,
|
||||
is_nextn: bool = False,
|
||||
) -> None:
|
||||
super().__init__(config, layer_id, quant_config, prefix, alt_stream, is_nextn)
|
||||
super().__init__(
|
||||
config,
|
||||
layer_id,
|
||||
quant_config,
|
||||
prefix,
|
||||
alt_stream,
|
||||
is_nextn,
|
||||
build_stages=False,
|
||||
)
|
||||
self._init_qwen4_exp_layer_extensions(config, layer_id, quant_config, prefix)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
**kwargs,
|
||||
):
|
||||
forward_batch = kwargs.get("forward_batch", None)
|
||||
|
||||
hidden_states, residual = self._prepare_qwen4_exp_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
ple_batch=kwargs.get("ple_batch"),
|
||||
hidden_states = self._prepare_attn_stage(
|
||||
hidden_states, forward_batch, kwargs.get("ple_batch")
|
||||
)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = self.linear_attn(hidden_states, forward_batch)
|
||||
|
||||
hidden_states, residual = self._prepare_qwen4_exp_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self._run_qwen4_exp_mlp(hidden_states, forward_batch)
|
||||
return self._postprocess_qwen4_exp_layer(hidden_states, residual, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
return self._run_ffn_stage(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class Qwen4ExpAttentionDecoderLayer(
|
||||
@@ -1642,7 +1652,15 @@ class Qwen4ExpAttentionDecoderLayer(
|
||||
is_nextn: bool = False,
|
||||
) -> None:
|
||||
config.attn_output_gate = True
|
||||
super().__init__(config, layer_id, quant_config, prefix, alt_stream, is_nextn)
|
||||
super().__init__(
|
||||
config,
|
||||
layer_id,
|
||||
quant_config,
|
||||
prefix,
|
||||
alt_stream,
|
||||
is_nextn,
|
||||
build_stages=False,
|
||||
)
|
||||
from sglang.srt.layers.attention.qsa.config import is_qwen_qsa
|
||||
from sglang.srt.layers.attention.qsa.glue import build_qsa_indexer
|
||||
|
||||
@@ -1779,15 +1797,11 @@ class Qwen4ExpAttentionDecoderLayer(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs: Any,
|
||||
):
|
||||
hidden_states, residual = self._prepare_qwen4_exp_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
ple_batch=kwargs.get("ple_batch"),
|
||||
hidden_states = self._prepare_attn_stage(
|
||||
hidden_states, forward_batch, kwargs.get("ple_batch")
|
||||
)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
@@ -1797,11 +1811,8 @@ class Qwen4ExpAttentionDecoderLayer(
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self._prepare_qwen4_exp_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self._run_qwen4_exp_mlp(hidden_states, forward_batch)
|
||||
return self._postprocess_qwen4_exp_layer(hidden_states, residual, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
return self._run_ffn_stage(hidden_states, forward_batch)
|
||||
|
||||
|
||||
ALL_DECODER_LAYER_TYPES = {
|
||||
@@ -1857,6 +1868,8 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
|
||||
if self.pp_group.is_last_rank
|
||||
else PPMissingLayer()
|
||||
)
|
||||
# The stack's terminal read mixes the streams down; there is no final norm.
|
||||
self._mix_streams = _MixStreams(self.hyper_connection_mixer)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -1871,18 +1884,17 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
residual = None
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
# Only the first stage embeds multimodal inputs; the last stage needs
|
||||
# them for the MTP draft prefill, so relay them with the hidden states.
|
||||
mm_input_embeds = pp_proxy_tensors.tensors.get("mm_input_embeds")
|
||||
if mm_input_embeds is not None:
|
||||
forward_batch.mm_input_embeds = mm_input_embeds
|
||||
# The hyper-connection streams ride in the widened hidden state;
|
||||
# there is no separate residual at a PP boundary (hc_hidden_size contract).
|
||||
residual = None
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
breakable_ple = (
|
||||
self.has_ple
|
||||
@@ -1916,10 +1928,9 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
|
||||
batch=ple_batch, forward_batch=forward_batch
|
||||
)
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
ple_batch=ple_batch,
|
||||
captured_last_layer_outputs=(
|
||||
@@ -1935,13 +1946,17 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
|
||||
_commit_ple_batch(batch=ple_batch, forward_batch=forward_batch)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
proxy_tensors = {"hidden_states": hidden_states}
|
||||
# The streams ride in the widened hidden state: the FFN write-back
|
||||
# already wrote them, so to_pp sends no separate residual.
|
||||
proxy_tensors = residual_batch.to_pp(hidden_states, forward_batch)
|
||||
if forward_batch.mm_input_embeds is not None:
|
||||
proxy_tensors["mm_input_embeds"] = forward_batch.mm_input_embeds
|
||||
return PPProxyTensors(proxy_tensors)
|
||||
return proxy_tensors
|
||||
|
||||
hc_hidden_states = hidden_states
|
||||
hidden_states, _ = self.hyper_connection_mixer.mix(hidden_states)
|
||||
hidden_states = residual_batch.final_norm(
|
||||
hidden_states, forward_batch, self._mix_streams
|
||||
)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
return hidden_states, hc_hidden_states
|
||||
|
||||
|
||||
Reference in New Issue
Block a user