[Refactor] Build the Hunyuan V4 decoder from stage boundaries (#42477)
This commit is contained in:
@@ -72,6 +72,9 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
PLAIN_ADD,
|
||||
PLAIN_RESIDUAL_OPS,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.ihc import (
|
||||
IHCState,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.mhc import (
|
||||
MHCState,
|
||||
)
|
||||
@@ -93,6 +96,7 @@ __all__ = [
|
||||
"FfnExit",
|
||||
"FfnInputFusion",
|
||||
"DeferredFinalize",
|
||||
"IHCState",
|
||||
"LayerResidualOps",
|
||||
"Layout",
|
||||
"MHCState",
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Gated hyper-connection residual streams."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
@dataclass
|
||||
class IHCState:
|
||||
"""A layer's residual as hc_mult gated streams: a stage reads its input by
|
||||
mixing the streams down with per-stream gates, and writes its output back
|
||||
scaled by the post gates the same read produced. Unlike the hyper-connection
|
||||
streams in mhc.py there is no combination matrix, so a read produces one
|
||||
coefficient rather than a pair. ``residual_ops()`` gives the reads and
|
||||
updates the layer's two stages declare. Parameters belong to the owning
|
||||
layer's modules; this state only holds the coefficient shared between a
|
||||
stage's read and its write-back.
|
||||
|
||||
Args:
|
||||
expand: Widens the layer stack's 2-D input into the stream
|
||||
representation, and passes an already widened residual through.
|
||||
attn_pre: The attention stage's read.
|
||||
ffn_pre: The FFN stage's read.
|
||||
attn_post: Writes the attention output back into the streams.
|
||||
ffn_post: Writes the FFN output back into the streams.
|
||||
post_pre: The attention write-back fused with the FFN read, or None to
|
||||
run the two in sequence.
|
||||
"""
|
||||
|
||||
expand: Callable
|
||||
attn_pre: Callable
|
||||
ffn_pre: Callable
|
||||
attn_post: Callable
|
||||
ffn_post: Callable
|
||||
post_pre: Optional[Callable] = None
|
||||
# Produced by a stage's read and consumed by that same stage's write-back.
|
||||
post_gate: Optional[torch.Tensor] = None
|
||||
|
||||
def read_attn_input(self, residual, out_norm: Optional[torch.nn.Module] = None):
|
||||
hidden_states, self.post_gate, residual = self.attn_pre(residual, out_norm)
|
||||
return hidden_states, residual
|
||||
|
||||
def update_and_read_ffn_input(
|
||||
self, hidden_states, residual, out_norm: Optional[torch.nn.Module] = None
|
||||
):
|
||||
if self.post_pre is not None:
|
||||
hidden_states, self.post_gate, residual = self.post_pre(
|
||||
hidden_states, residual, self.post_gate, out_norm
|
||||
)
|
||||
return hidden_states, residual
|
||||
residual = self.apply_attn_post(hidden_states, residual)
|
||||
hidden_states, self.post_gate, residual = self.ffn_pre(residual, out_norm)
|
||||
return hidden_states, residual
|
||||
|
||||
def apply_attn_post(self, hidden_states, residual):
|
||||
return self.attn_post(hidden_states, residual, self.post_gate)
|
||||
|
||||
def apply_ffn_post(self, hidden_states, residual):
|
||||
return self.ffn_post(hidden_states, residual, self.post_gate)
|
||||
|
||||
def clear_coefficients(self):
|
||||
self.post_gate = None
|
||||
|
||||
def slice_residual_attn_tp(self, residual):
|
||||
parallel = get_parallel()
|
||||
rank, size = parallel.attn_tp_rank, parallel.attn_tp_size
|
||||
self.post_gate = self.post_gate.tensor_split(size)[rank]
|
||||
return residual.tensor_split(size)[rank]
|
||||
|
||||
def gather_residual_attn_tp(self, residual):
|
||||
raise NotImplementedError(
|
||||
"Unsupported: iHC post-gate allgather is not implemented."
|
||||
)
|
||||
|
||||
def residual_ops(self) -> LayerResidualOps:
|
||||
return LayerResidualOps(
|
||||
attn_readout=_AttnReadout(self),
|
||||
attn_update=_AttnUpdate(self),
|
||||
ffn_readout=_FfnReadout(self),
|
||||
ffn_update=_FfnUpdate(self),
|
||||
)
|
||||
|
||||
|
||||
class _AttnReadout:
|
||||
"""The gated mix of the streams and the input norm, from streams that
|
||||
already hold the previous layer's output: an iHC layer takes its input
|
||||
written back."""
|
||||
|
||||
is_plain_norm = False
|
||||
reads_before_dp_gather = False
|
||||
|
||||
def __init__(self, state: IHCState):
|
||||
self.state = state
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
return self.state.expand(hidden_states)
|
||||
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
if quant_format:
|
||||
raise NotImplementedError(f"an iHC attention input in {quant_format=}")
|
||||
return self.state.read_attn_input(residual, out_norm=norm)
|
||||
|
||||
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
|
||||
raise NotImplementedError("an iHC layer takes its input written back")
|
||||
|
||||
|
||||
class _AttnUpdate:
|
||||
"""The gated write-back with the coefficient the attention's read produced.
|
||||
It is not a plain add, so it runs only once the sum it writes in is
|
||||
complete."""
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
outlives_layer = False
|
||||
|
||||
def __init__(self, state: IHCState):
|
||||
self.state = state
|
||||
|
||||
def update(self, hidden_states, residual):
|
||||
return self.state.apply_attn_post(hidden_states, residual)
|
||||
|
||||
def slice_residual_attn_tp(self, residual):
|
||||
return self.state.slice_residual_attn_tp(residual)
|
||||
|
||||
def gather_residual_attn_tp(self, residual):
|
||||
return self.state.gather_residual_attn_tp(residual)
|
||||
|
||||
|
||||
class _FfnReadout:
|
||||
"""The attention output's write-back and the FFN input's gated mix and
|
||||
norm, fused in post_pre when the layer provides it."""
|
||||
|
||||
is_plain_norm = False
|
||||
reads_before_dp_gather = False
|
||||
|
||||
def __init__(self, state: IHCState):
|
||||
self.state = state
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
return self.state.expand(hidden_states)
|
||||
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
raise NotImplementedError("an iHC FFN input read without its attention")
|
||||
|
||||
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
|
||||
if not isinstance(update, _AttnUpdate) or update.state is not self.state:
|
||||
raise NotImplementedError(f"an iHC FFN input after {update=}")
|
||||
return self.state.update_and_read_ffn_input(
|
||||
hidden_states, residual, out_norm=norm
|
||||
)
|
||||
|
||||
|
||||
class _FfnUpdate:
|
||||
"""The gated write-back with the coefficient the FFN input's read produced,
|
||||
which this layer runs itself. The streams stay widened: unlike mhc.py the
|
||||
layer stack's terminal read is a learned head rather than a contraction,
|
||||
so it belongs to the final norm."""
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
outlives_layer = False
|
||||
|
||||
def __init__(self, state: IHCState):
|
||||
self.state = state
|
||||
|
||||
def update(self, hidden_states, residual):
|
||||
hidden_states = self.state.apply_ffn_post(hidden_states, residual)
|
||||
self.state.clear_coefficients()
|
||||
return hidden_states
|
||||
|
||||
def slice_residual_attn_tp(self, residual):
|
||||
return self.state.slice_residual_attn_tp(residual)
|
||||
|
||||
def gather_residual_attn_tp(self, residual):
|
||||
return self.state.gather_residual_attn_tp(residual)
|
||||
@@ -7,7 +7,14 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
|
||||
from sglang.srt.layers.layer_boundary import AttentionInputs, get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
IHCState,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
get_attn_tp_context,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear, ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -342,7 +349,8 @@ class HYV4Attention(DeepseekV2AttentionMLA):
|
||||
),
|
||||
max_position_embeddings=config.max_position_embeddings,
|
||||
quant_config=quant_config,
|
||||
reduce_results=True,
|
||||
# The FFN input boundary completes this stage's sum.
|
||||
reduce_results=False,
|
||||
layer_id=layer_id,
|
||||
prefix=prefix,
|
||||
alt_stream=alt_stream,
|
||||
@@ -535,6 +543,77 @@ class HYV4Attention(DeepseekV2AttentionMLA):
|
||||
return method
|
||||
|
||||
|
||||
class _HeadNorm:
|
||||
"""The layer stack's terminal read: the learned head mixes the streams down
|
||||
and fuses the final norm into the same kernel."""
|
||||
|
||||
def __init__(self, hc_head, norm):
|
||||
self.hc_head = hc_head
|
||||
self.norm = norm
|
||||
|
||||
def __call__(self, hidden_states):
|
||||
return self.hc_head(hidden_states, self.norm)
|
||||
|
||||
|
||||
def _build_stages(
|
||||
hc_attn_layer,
|
||||
hc_mlp_layer,
|
||||
input_layernorm,
|
||||
post_attention_layernorm,
|
||||
*,
|
||||
qkv_latent_func,
|
||||
layer_id,
|
||||
config,
|
||||
):
|
||||
"""The attention and FFN stage boundaries of one iHC decoder layer."""
|
||||
|
||||
def post_pre(hidden_states, residual, post_gate, out_norm):
|
||||
return hc_attn_layer.post_pre(
|
||||
hidden_states, residual, post_gate, hc_mlp_layer, out_norm
|
||||
)
|
||||
|
||||
residual = IHCState(
|
||||
expand=hc_attn_layer.prepare_input,
|
||||
attn_pre=hc_attn_layer.pre,
|
||||
ffn_pre=hc_mlp_layer.pre,
|
||||
attn_post=hc_attn_layer.post,
|
||||
ffn_post=hc_mlp_layer.post,
|
||||
post_pre=post_pre,
|
||||
).residual_ops()
|
||||
|
||||
layer_types = config.mlp_layer_types
|
||||
is_sparse = layer_types[layer_id] != "dense"
|
||||
next_id, previous_id = layer_id + 1, layer_id - 1
|
||||
return make_stages(
|
||||
(
|
||||
declare_attn(read=residual.attn_readout, update=residual.attn_update),
|
||||
input_layernorm,
|
||||
{"qkv_latent_func": qkv_latent_func},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_sparse,
|
||||
next_layer_sparse=(
|
||||
next_id < len(layer_types) and layer_types[next_id] != "dense"
|
||||
),
|
||||
read=residual.ffn_readout,
|
||||
update=residual.ffn_update,
|
||||
),
|
||||
post_attention_layernorm,
|
||||
),
|
||||
previous=(
|
||||
declare_ffn(
|
||||
sparse=layer_types[previous_id] != "dense",
|
||||
next_layer_sparse=is_sparse,
|
||||
update=residual.ffn_update,
|
||||
)
|
||||
if layer_id != 0
|
||||
else None
|
||||
),
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
|
||||
class HYV4DecoderLayer(nn.Module):
|
||||
def __init__(self, config, layer_id, quant_config=None, prefix="", alt_stream=None):
|
||||
super().__init__()
|
||||
@@ -550,6 +629,8 @@ class HYV4DecoderLayer(nn.Module):
|
||||
config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
reduce_results=False,
|
||||
allow_fused_down=False,
|
||||
)
|
||||
else:
|
||||
self.mlp = DeepseekV2MoE(
|
||||
@@ -558,11 +639,21 @@ class HYV4DecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
alt_stream=alt_stream,
|
||||
reduce_results=False,
|
||||
)
|
||||
if hasattr(self.mlp, "shared_experts"):
|
||||
self.mlp.shared_experts.swiglu_limit = None
|
||||
self.hc_attn_layer = HYV4HCLayer(config, f"{prefix}.hc_attn_layer")
|
||||
self.hc_mlp_layer = HYV4HCLayer(config, f"{prefix}.hc_mlp_layer")
|
||||
self.attn_boundary, self.ffn_boundary = _build_stages(
|
||||
self.hc_attn_layer,
|
||||
self.hc_mlp_layer,
|
||||
self.input_layernorm,
|
||||
self.post_attention_layernorm,
|
||||
qkv_latent_func=self.self_attn.prepare_qkv_latent,
|
||||
layer_id=layer_id,
|
||||
config=config,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -572,15 +663,7 @@ class HYV4DecoderLayer(nn.Module):
|
||||
zero_allocator,
|
||||
prev_topk_indices=None,
|
||||
):
|
||||
hidden_states = self.hc_attn_layer.prepare_input(hidden_states)
|
||||
hidden_states, post, residual = self.hc_attn_layer.pre(
|
||||
hidden_states, self.input_layernorm
|
||||
)
|
||||
get_attn_tp_context().set_attn_inputs(
|
||||
AttentionInputs(
|
||||
hidden_states, forward_batch, self.self_attn.prepare_qkv_latent
|
||||
)
|
||||
)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
try:
|
||||
hidden_states = self.self_attn(
|
||||
positions,
|
||||
@@ -595,18 +678,13 @@ class HYV4DecoderLayer(nn.Module):
|
||||
hidden_states, topk_indices = hidden_states
|
||||
else:
|
||||
topk_indices = None
|
||||
hidden_states, post, residual = self.hc_attn_layer.post_pre(
|
||||
hidden_states,
|
||||
residual,
|
||||
post,
|
||||
self.hc_mlp_layer,
|
||||
self.post_attention_layernorm,
|
||||
)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
if isinstance(self.mlp, DeepseekV2MoE):
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = self.hc_mlp_layer.post(hidden_states, residual, post)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
return hidden_states, topk_indices
|
||||
|
||||
|
||||
@@ -639,11 +717,14 @@ class HYV4Model(nn.Module):
|
||||
)
|
||||
self.hc_head = HYV4HCHeadLayer(config, f"{prefix}.hc_head")
|
||||
self.norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
|
||||
# The head mixes the streams down and fuses the final norm into one kernel.
|
||||
self._head_norm = _HeadNorm(self.hc_head, self.norm)
|
||||
|
||||
def forward(self, input_ids, positions, forward_batch, input_embeds=None):
|
||||
hidden_states = (
|
||||
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
|
||||
)
|
||||
residual_batch.start(forward_batch)
|
||||
zero_allocator = BumpAllocator(
|
||||
buffer_size=2 * len(self.layers),
|
||||
dtype=torch.float32,
|
||||
@@ -660,7 +741,7 @@ class HYV4Model(nn.Module):
|
||||
)
|
||||
topk_share.update(topk_indices)
|
||||
topk_share.publish()
|
||||
return self.hc_head(hidden_states, self.norm)
|
||||
return residual_batch.final_norm(hidden_states, forward_batch, self._head_norm)
|
||||
|
||||
|
||||
class HYV4ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
|
||||
|
||||
@@ -5,7 +5,13 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
|
||||
from sglang.srt.layers.layer_boundary import AttentionInputs, get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
get_attn_tp_context,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
@@ -61,9 +67,19 @@ class HYV4MTPDecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.mlp",
|
||||
alt_stream=alt_stream,
|
||||
is_nextn=True,
|
||||
reduce_results=False,
|
||||
)
|
||||
if hasattr(self.mlp, "shared_experts"):
|
||||
self.mlp.shared_experts.swiglu_limit = None
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
{"qkv_latent_func": self.self_attn.prepare_qkv_latent},
|
||||
),
|
||||
(declare_ffn(sparse=True), self.post_attention_layernorm),
|
||||
terminal=True,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -73,13 +89,7 @@ class HYV4MTPDecoderLayer(nn.Module):
|
||||
zero_allocator,
|
||||
prev_topk_indices=None,
|
||||
):
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
get_attn_tp_context().set_attn_inputs(
|
||||
AttentionInputs(
|
||||
hidden_states, forward_batch, self.self_attn.prepare_qkv_latent
|
||||
)
|
||||
)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
try:
|
||||
hidden_states = self.self_attn(
|
||||
positions,
|
||||
@@ -94,9 +104,11 @@ class HYV4MTPDecoderLayer(nn.Module):
|
||||
hidden_states, topk_indices = hidden_states
|
||||
else:
|
||||
topk_indices = None
|
||||
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
return hidden_states, residual, topk_indices
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
return hidden_states, topk_indices
|
||||
|
||||
|
||||
class HYV4ModelNextN(nn.Module):
|
||||
@@ -140,8 +152,9 @@ class HYV4ModelNextN(nn.Module):
|
||||
dtype=torch.float32,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
residual_batch.start(forward_batch)
|
||||
topk_share = IndexTopKShareState.from_mtp_carry(forward_batch)
|
||||
hidden_states, residual, topk_indices = self.decoder(
|
||||
hidden_states, topk_indices = self.decoder(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
@@ -150,9 +163,11 @@ class HYV4ModelNextN(nn.Module):
|
||||
)
|
||||
topk_share.update(topk_indices)
|
||||
topk_share.publish()
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
return hidden_states
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.final_norm(
|
||||
hidden_states, forward_batch, self.shared_head.norm
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user