[Refactor] Build the Hunyuan V4 decoder from stage boundaries (#42477)

This commit is contained in:
Cheng Wan
2026-10-04 16:38:03 -07:00
committed by GitHub
parent f5ccc4d7d9
commit 92d60351e2
5 changed files with 531 additions and 34 deletions
@@ -72,6 +72,9 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import (
PLAIN_ADD,
PLAIN_RESIDUAL_OPS,
)
from sglang.srt.layers.layer_boundary.residual.ihc import (
IHCState,
)
from sglang.srt.layers.layer_boundary.residual.mhc import (
MHCState,
)
@@ -93,6 +96,7 @@ __all__ = [
"FfnExit",
"FfnInputFusion",
"DeferredFinalize",
"IHCState",
"LayerResidualOps",
"Layout",
"MHCState",
@@ -0,0 +1,192 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Gated hyper-connection residual streams."""
from dataclasses import dataclass
from typing import Callable, Optional
import torch
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
from sglang.srt.runtime_context import get_parallel
@dataclass
class IHCState:
"""A layer's residual as hc_mult gated streams: a stage reads its input by
mixing the streams down with per-stream gates, and writes its output back
scaled by the post gates the same read produced. Unlike the hyper-connection
streams in mhc.py there is no combination matrix, so a read produces one
coefficient rather than a pair. ``residual_ops()`` gives the reads and
updates the layer's two stages declare. Parameters belong to the owning
layer's modules; this state only holds the coefficient shared between a
stage's read and its write-back.
Args:
expand: Widens the layer stack's 2-D input into the stream
representation, and passes an already widened residual through.
attn_pre: The attention stage's read.
ffn_pre: The FFN stage's read.
attn_post: Writes the attention output back into the streams.
ffn_post: Writes the FFN output back into the streams.
post_pre: The attention write-back fused with the FFN read, or None to
run the two in sequence.
"""
expand: Callable
attn_pre: Callable
ffn_pre: Callable
attn_post: Callable
ffn_post: Callable
post_pre: Optional[Callable] = None
# Produced by a stage's read and consumed by that same stage's write-back.
post_gate: Optional[torch.Tensor] = None
def read_attn_input(self, residual, out_norm: Optional[torch.nn.Module] = None):
hidden_states, self.post_gate, residual = self.attn_pre(residual, out_norm)
return hidden_states, residual
def update_and_read_ffn_input(
self, hidden_states, residual, out_norm: Optional[torch.nn.Module] = None
):
if self.post_pre is not None:
hidden_states, self.post_gate, residual = self.post_pre(
hidden_states, residual, self.post_gate, out_norm
)
return hidden_states, residual
residual = self.apply_attn_post(hidden_states, residual)
hidden_states, self.post_gate, residual = self.ffn_pre(residual, out_norm)
return hidden_states, residual
def apply_attn_post(self, hidden_states, residual):
return self.attn_post(hidden_states, residual, self.post_gate)
def apply_ffn_post(self, hidden_states, residual):
return self.ffn_post(hidden_states, residual, self.post_gate)
def clear_coefficients(self):
self.post_gate = None
def slice_residual_attn_tp(self, residual):
parallel = get_parallel()
rank, size = parallel.attn_tp_rank, parallel.attn_tp_size
self.post_gate = self.post_gate.tensor_split(size)[rank]
return residual.tensor_split(size)[rank]
def gather_residual_attn_tp(self, residual):
raise NotImplementedError(
"Unsupported: iHC post-gate allgather is not implemented."
)
def residual_ops(self) -> LayerResidualOps:
return LayerResidualOps(
attn_readout=_AttnReadout(self),
attn_update=_AttnUpdate(self),
ffn_readout=_FfnReadout(self),
ffn_update=_FfnUpdate(self),
)
class _AttnReadout:
"""The gated mix of the streams and the input norm, from streams that
already hold the previous layer's output: an iHC layer takes its input
written back."""
is_plain_norm = False
reads_before_dp_gather = False
def __init__(self, state: IHCState):
self.state = state
def init_residual(self, hidden_states):
return self.state.expand(hidden_states)
def read(self, residual, norm, quant_format="", post_residual_addition=None):
if quant_format:
raise NotImplementedError(f"an iHC attention input in {quant_format=}")
return self.state.read_attn_input(residual, out_norm=norm)
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
raise NotImplementedError("an iHC layer takes its input written back")
class _AttnUpdate:
"""The gated write-back with the coefficient the attention's read produced.
It is not a plain add, so it runs only once the sum it writes in is
complete."""
is_plain_add = False
applied_at_exit = False
outlives_layer = False
def __init__(self, state: IHCState):
self.state = state
def update(self, hidden_states, residual):
return self.state.apply_attn_post(hidden_states, residual)
def slice_residual_attn_tp(self, residual):
return self.state.slice_residual_attn_tp(residual)
def gather_residual_attn_tp(self, residual):
return self.state.gather_residual_attn_tp(residual)
class _FfnReadout:
"""The attention output's write-back and the FFN input's gated mix and
norm, fused in post_pre when the layer provides it."""
is_plain_norm = False
reads_before_dp_gather = False
def __init__(self, state: IHCState):
self.state = state
def init_residual(self, hidden_states):
return self.state.expand(hidden_states)
def read(self, residual, norm, quant_format="", post_residual_addition=None):
raise NotImplementedError("an iHC FFN input read without its attention")
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
if not isinstance(update, _AttnUpdate) or update.state is not self.state:
raise NotImplementedError(f"an iHC FFN input after {update=}")
return self.state.update_and_read_ffn_input(
hidden_states, residual, out_norm=norm
)
class _FfnUpdate:
"""The gated write-back with the coefficient the FFN input's read produced,
which this layer runs itself. The streams stay widened: unlike mhc.py the
layer stack's terminal read is a learned head rather than a contraction,
so it belongs to the final norm."""
is_plain_add = False
applied_at_exit = True
outlives_layer = False
def __init__(self, state: IHCState):
self.state = state
def update(self, hidden_states, residual):
hidden_states = self.state.apply_ffn_post(hidden_states, residual)
self.state.clear_coefficients()
return hidden_states
def slice_residual_attn_tp(self, residual):
return self.state.slice_residual_attn_tp(residual)
def gather_residual_attn_tp(self, residual):
return self.state.gather_residual_attn_tp(residual)
+101 -20
View File
@@ -7,7 +7,14 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
from sglang.srt.layers.layer_boundary import AttentionInputs, get_attn_tp_context
from sglang.srt.layers.layer_boundary import (
IHCState,
declare_attn,
declare_ffn,
get_attn_tp_context,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ColumnParallelLinear, ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -342,7 +349,8 @@ class HYV4Attention(DeepseekV2AttentionMLA):
),
max_position_embeddings=config.max_position_embeddings,
quant_config=quant_config,
reduce_results=True,
# The FFN input boundary completes this stage's sum.
reduce_results=False,
layer_id=layer_id,
prefix=prefix,
alt_stream=alt_stream,
@@ -535,6 +543,77 @@ class HYV4Attention(DeepseekV2AttentionMLA):
return method
class _HeadNorm:
"""The layer stack's terminal read: the learned head mixes the streams down
and fuses the final norm into the same kernel."""
def __init__(self, hc_head, norm):
self.hc_head = hc_head
self.norm = norm
def __call__(self, hidden_states):
return self.hc_head(hidden_states, self.norm)
def _build_stages(
hc_attn_layer,
hc_mlp_layer,
input_layernorm,
post_attention_layernorm,
*,
qkv_latent_func,
layer_id,
config,
):
"""The attention and FFN stage boundaries of one iHC decoder layer."""
def post_pre(hidden_states, residual, post_gate, out_norm):
return hc_attn_layer.post_pre(
hidden_states, residual, post_gate, hc_mlp_layer, out_norm
)
residual = IHCState(
expand=hc_attn_layer.prepare_input,
attn_pre=hc_attn_layer.pre,
ffn_pre=hc_mlp_layer.pre,
attn_post=hc_attn_layer.post,
ffn_post=hc_mlp_layer.post,
post_pre=post_pre,
).residual_ops()
layer_types = config.mlp_layer_types
is_sparse = layer_types[layer_id] != "dense"
next_id, previous_id = layer_id + 1, layer_id - 1
return make_stages(
(
declare_attn(read=residual.attn_readout, update=residual.attn_update),
input_layernorm,
{"qkv_latent_func": qkv_latent_func},
),
(
declare_ffn(
sparse=is_sparse,
next_layer_sparse=(
next_id < len(layer_types) and layer_types[next_id] != "dense"
),
read=residual.ffn_readout,
update=residual.ffn_update,
),
post_attention_layernorm,
),
previous=(
declare_ffn(
sparse=layer_types[previous_id] != "dense",
next_layer_sparse=is_sparse,
update=residual.ffn_update,
)
if layer_id != 0
else None
),
terminal=layer_id == config.num_hidden_layers - 1,
)
class HYV4DecoderLayer(nn.Module):
def __init__(self, config, layer_id, quant_config=None, prefix="", alt_stream=None):
super().__init__()
@@ -550,6 +629,8 @@ class HYV4DecoderLayer(nn.Module):
config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
reduce_results=False,
allow_fused_down=False,
)
else:
self.mlp = DeepseekV2MoE(
@@ -558,11 +639,21 @@ class HYV4DecoderLayer(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.mlp",
alt_stream=alt_stream,
reduce_results=False,
)
if hasattr(self.mlp, "shared_experts"):
self.mlp.shared_experts.swiglu_limit = None
self.hc_attn_layer = HYV4HCLayer(config, f"{prefix}.hc_attn_layer")
self.hc_mlp_layer = HYV4HCLayer(config, f"{prefix}.hc_mlp_layer")
self.attn_boundary, self.ffn_boundary = _build_stages(
self.hc_attn_layer,
self.hc_mlp_layer,
self.input_layernorm,
self.post_attention_layernorm,
qkv_latent_func=self.self_attn.prepare_qkv_latent,
layer_id=layer_id,
config=config,
)
def forward(
self,
@@ -572,15 +663,7 @@ class HYV4DecoderLayer(nn.Module):
zero_allocator,
prev_topk_indices=None,
):
hidden_states = self.hc_attn_layer.prepare_input(hidden_states)
hidden_states, post, residual = self.hc_attn_layer.pre(
hidden_states, self.input_layernorm
)
get_attn_tp_context().set_attn_inputs(
AttentionInputs(
hidden_states, forward_batch, self.self_attn.prepare_qkv_latent
)
)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
try:
hidden_states = self.self_attn(
positions,
@@ -595,18 +678,13 @@ class HYV4DecoderLayer(nn.Module):
hidden_states, topk_indices = hidden_states
else:
topk_indices = None
hidden_states, post, residual = self.hc_attn_layer.post_pre(
hidden_states,
residual,
post,
self.hc_mlp_layer,
self.post_attention_layernorm,
)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
if isinstance(self.mlp, DeepseekV2MoE):
hidden_states = self.mlp(hidden_states, forward_batch)
else:
hidden_states = self.mlp(hidden_states)
hidden_states = self.hc_mlp_layer.post(hidden_states, residual, post)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states, topk_indices
@@ -639,11 +717,14 @@ class HYV4Model(nn.Module):
)
self.hc_head = HYV4HCHeadLayer(config, f"{prefix}.hc_head")
self.norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
# The head mixes the streams down and fuses the final norm into one kernel.
self._head_norm = _HeadNorm(self.hc_head, self.norm)
def forward(self, input_ids, positions, forward_batch, input_embeds=None):
hidden_states = (
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
)
residual_batch.start(forward_batch)
zero_allocator = BumpAllocator(
buffer_size=2 * len(self.layers),
dtype=torch.float32,
@@ -660,7 +741,7 @@ class HYV4Model(nn.Module):
)
topk_share.update(topk_indices)
topk_share.publish()
return self.hc_head(hidden_states, self.norm)
return residual_batch.final_norm(hidden_states, forward_batch, self._head_norm)
class HYV4ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
+29 -14
View File
@@ -5,7 +5,13 @@ import torch
from torch import nn
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
from sglang.srt.layers.layer_boundary import AttentionInputs, get_attn_tp_context
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
get_attn_tp_context,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.vocab_parallel_embedding import (
@@ -61,9 +67,19 @@ class HYV4MTPDecoderLayer(nn.Module):
prefix=f"{prefix}.mlp",
alt_stream=alt_stream,
is_nextn=True,
reduce_results=False,
)
if hasattr(self.mlp, "shared_experts"):
self.mlp.shared_experts.swiglu_limit = None
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(),
self.input_layernorm,
{"qkv_latent_func": self.self_attn.prepare_qkv_latent},
),
(declare_ffn(sparse=True), self.post_attention_layernorm),
terminal=True,
)
def forward(
self,
@@ -73,13 +89,7 @@ class HYV4MTPDecoderLayer(nn.Module):
zero_allocator,
prev_topk_indices=None,
):
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
get_attn_tp_context().set_attn_inputs(
AttentionInputs(
hidden_states, forward_batch, self.self_attn.prepare_qkv_latent
)
)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
try:
hidden_states = self.self_attn(
positions,
@@ -94,9 +104,11 @@ class HYV4MTPDecoderLayer(nn.Module):
hidden_states, topk_indices = hidden_states
else:
topk_indices = None
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states, forward_batch)
return hidden_states, residual, topk_indices
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states, topk_indices
class HYV4ModelNextN(nn.Module):
@@ -140,8 +152,9 @@ class HYV4ModelNextN(nn.Module):
dtype=torch.float32,
device=hidden_states.device,
)
residual_batch.start(forward_batch)
topk_share = IndexTopKShareState.from_mtp_carry(forward_batch)
hidden_states, residual, topk_indices = self.decoder(
hidden_states, topk_indices = self.decoder(
positions,
hidden_states,
forward_batch,
@@ -150,9 +163,11 @@ class HYV4ModelNextN(nn.Module):
)
topk_share.update(topk_indices)
topk_share.publish()
if forward_batch.forward_mode.is_idle():
return hidden_states
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.final_norm(
hidden_states, forward_batch, self.shared_head.norm
)
return hidden_states