[Refactor] Build the ZAYA1, IQuest-Q1 and Gemma 4 decoders from stage boundaries (#42311)
This commit is contained in:
@@ -46,6 +46,7 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
NORM_QUANT_READOUT,
|
||||
NORM_READOUT,
|
||||
PLAIN_ADD,
|
||||
REPLACE_AT_EXIT,
|
||||
)
|
||||
from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
@@ -156,9 +157,11 @@ def _resolve_ffn(
|
||||
*((TokenAxis.ATTN_CP,) if on_cp_shards else ()), axis_sizes=axes
|
||||
)
|
||||
)
|
||||
# An FFN that writes the next stream itself hands on a complete output.
|
||||
complete = on_rank_rows or update is REPLACE_AT_EXIT
|
||||
produced = (
|
||||
OutputContract(rows, update=update, transform=output_transform)
|
||||
if on_rank_rows
|
||||
if complete
|
||||
else OutputContract(
|
||||
rows,
|
||||
group=group,
|
||||
|
||||
@@ -280,6 +280,26 @@ class PlainAdd:
|
||||
return attn_tp_gather(residual)
|
||||
|
||||
|
||||
class ReplaceAtExit:
|
||||
"""The producer computes the next stream itself (its own add, norms and
|
||||
scaling, in whatever kernels it uses) and writes it at its exit; the
|
||||
previous residual is dropped. The stream is complete: the producer has its
|
||||
boundary complete each part's sum first (StageBoundary.sum_part)."""
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
outlives_layer = True
|
||||
|
||||
def update(self, hidden_states, residual):
|
||||
return hidden_states
|
||||
|
||||
def slice_residual_attn_tp(self, residual):
|
||||
return attn_tp_slice(residual)
|
||||
|
||||
def gather_residual_attn_tp(self, residual):
|
||||
return attn_tp_gather(residual)
|
||||
|
||||
|
||||
class Fp8Input(Enum):
|
||||
"""Additional input forms a projection can consume after a fused norm."""
|
||||
|
||||
@@ -390,6 +410,7 @@ class UnfusedNormReadout(NormReadout):
|
||||
|
||||
|
||||
PLAIN_ADD = PlainAdd()
|
||||
REPLACE_AT_EXIT = ReplaceAtExit()
|
||||
NORM_QUANT_READOUT = NormQuantReadout()
|
||||
NORM_READOUT = NormReadout()
|
||||
UNFUSED_NORM_READOUT = UnfusedNormReadout()
|
||||
|
||||
@@ -35,6 +35,12 @@ def stream_of(forward_batch):
|
||||
return stream
|
||||
|
||||
|
||||
def written_residual(forward_batch):
|
||||
"""The residual the last prepare wrote, for a stage whose compute reads it
|
||||
too (e.g. to write the next stream at its exit). Borrowed: do not modify."""
|
||||
return stream_of(forward_batch).residual
|
||||
|
||||
|
||||
def complete_output(hidden_states, forward_batch):
|
||||
"""Complete the contribution while retaining its pending residual update."""
|
||||
return stream_of(forward_batch).complete(hidden_states)
|
||||
|
||||
@@ -30,10 +30,12 @@ from sglang.srt.layers.layer_boundary.contracts import (
|
||||
StageKind,
|
||||
StagePath,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.ops import attn_tp_gather_input
|
||||
from sglang.srt.layers.layer_boundary.layout import SumGroup
|
||||
from sglang.srt.layers.layer_boundary.ops import attn_tp_gather_input, sum_output
|
||||
from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
PLAIN_ADD,
|
||||
REPLACE_AT_EXIT,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.batch import stream_of
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum, ResidualStream
|
||||
@@ -262,6 +264,20 @@ class StageBoundary:
|
||||
hidden_states, stream_of(forward_batch), forward_batch
|
||||
)
|
||||
|
||||
def sum_part(self, hidden_states, forward_batch, group: SumGroup):
|
||||
"""Complete the sum over ``group`` that one part of this FFN's output
|
||||
owes.
|
||||
|
||||
For an FFN that builds the next stream itself (REPLACE_AT_EXIT) from
|
||||
several complete parts, normalizing each before combining them; the
|
||||
stream it hands to its exit is then complete.
|
||||
"""
|
||||
if self.declaration.update is not REPLACE_AT_EXIT:
|
||||
raise RuntimeError(
|
||||
"only an FFN that writes the next stream itself sums its parts"
|
||||
)
|
||||
return sum_output(hidden_states, group, forward_batch, may_quantize=False)
|
||||
|
||||
def branch_input(self, source, hidden_states, forward_batch):
|
||||
hidden_states, stream = branch.branch_input(
|
||||
self.plan,
|
||||
|
||||
@@ -83,6 +83,7 @@ class Gemma3MLP(nn.Module):
|
||||
hidden_activation: str,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
@@ -97,6 +98,7 @@ class Gemma3MLP(nn.Module):
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
)
|
||||
if hidden_activation != "gelu_pytorch_tanh":
|
||||
|
||||
@@ -33,6 +33,16 @@ from sglang.kernels.ops.moe.gemma4_routing import (
|
||||
gemma4_fused_routing,
|
||||
gemma_routing_post_topk,
|
||||
)
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
SumGroup,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import REPLACE_AT_EXIT
|
||||
from sglang.srt.layers.layernorm import Gemma4RMSNorm, RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -219,6 +229,7 @@ class Gemma4MoE(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
activation: str = "gelu",
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.layer_id = layer_id
|
||||
@@ -277,7 +288,7 @@ class Gemma4MoE(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("experts", prefix),
|
||||
activation=activation,
|
||||
reduce_results=True,
|
||||
reduce_results=reduce_results,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -352,6 +363,7 @@ class Gemma4Attention(nn.Module):
|
||||
hidden_size,
|
||||
bias=config.attention_bias,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -578,6 +590,7 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
hidden_activation=config.hidden_activation,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -631,6 +644,7 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("moe", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
self.post_feedforward_layernorm_1 = RMSNorm(
|
||||
@@ -652,6 +666,30 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
self.register_buffer("layer_scalar", torch.ones(1), persistent=True)
|
||||
self.has_ple = self.hidden_size_per_layer_input > 0
|
||||
self.prefix = prefix
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(
|
||||
output_transform=OutputTransform(self.post_attention_layernorm)
|
||||
),
|
||||
self.input_layernorm,
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.enable_moe_block,
|
||||
next_layer_sparse=self.enable_moe_block,
|
||||
update=REPLACE_AT_EXIT,
|
||||
),
|
||||
self.pre_feedforward_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=self.enable_moe_block,
|
||||
next_layer_sparse=self.enable_moe_block,
|
||||
update=REPLACE_AT_EXIT,
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -659,47 +697,63 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
per_layer_input: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
capture_output=None,
|
||||
**kwargs,
|
||||
) -> tuple[
|
||||
torch.FloatTensor, Optional[tuple[torch.FloatTensor, torch.FloatTensor]]
|
||||
]:
|
||||
) -> torch.Tensor:
|
||||
# Gemma4 residual pattern following JAX implementation:
|
||||
# 1. input_norm(x) -> attn -> post_attn_norm -> ADD residual
|
||||
# 2. pre_ff_norm -> mlp -> post_ff_norm -> ADD residual
|
||||
#
|
||||
# Optimization: fuse "post_attn_norm(h) + residual; pre_ff_norm(...)"
|
||||
# into "post_attn_norm(h); pre_ff_norm(h, residual)" using
|
||||
# gemma_fused_add_rmsnorm which computes:
|
||||
# The FFN input runs post_attn_norm(h), then pre_ff_norm(h, residual)
|
||||
# with gemma_fused_add_rmsnorm, which computes:
|
||||
# residual = h + residual (in-place)
|
||||
# h = gemma_norm(residual)
|
||||
residual = hidden_states
|
||||
|
||||
# Apply input layernorm
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch, capture=capture_output
|
||||
)
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states = self.post_attention_layernorm(hidden_states)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
residual = residual_batch.written_residual(forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self._next_stream(
|
||||
hidden_states, residual, per_layer_input, forward_batch
|
||||
)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
def _next_stream(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
per_layer_input: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
"""The next stream from the FFN input and the residual the FFN input
|
||||
wrote. Each part's TP sum is completed by the FFN boundary before its
|
||||
post-FFN norm."""
|
||||
|
||||
if self.enable_moe_block:
|
||||
# Fuse: hidden_states + residual -> residual; pre_ff_norm(residual) -> hidden_states
|
||||
# Also need raw (unfused) residual for router and pre_ff_norm_2
|
||||
hidden_states, residual = self.pre_feedforward_layernorm(
|
||||
hidden_states, residual
|
||||
)
|
||||
# For MoE: router and pre_ff_norm_2 need the unfused residual
|
||||
# (which is now updated to post_attn_out + old_residual)
|
||||
moe_input = residual
|
||||
|
||||
# Dense MLP branch
|
||||
hidden_states_1 = self.mlp(hidden_states)
|
||||
hidden_states_1 = self.ffn_boundary.sum_part(
|
||||
self.mlp(hidden_states), forward_batch, SumGroup.TP
|
||||
)
|
||||
|
||||
# MoE branch: router sees residual (= post_attn_out + old_residual)
|
||||
router_logits = self.router(moe_input)
|
||||
hidden_states_2 = self.pre_feedforward_layernorm_2(moe_input)
|
||||
hidden_states_2 = self.moe(hidden_states_2, router_logits)
|
||||
hidden_states_2 = self.ffn_boundary.sum_part(
|
||||
self.moe(hidden_states_2, router_logits),
|
||||
forward_batch,
|
||||
SumGroup.MOE_OUTPUT,
|
||||
)
|
||||
|
||||
# Fused: (rmsnorm(rmsnorm(h1,w1) + rmsnorm(h2,w2), w3) + residual) * scalar
|
||||
if (
|
||||
@@ -722,7 +776,7 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
norm2.variance_epsilon,
|
||||
norm3.variance_epsilon,
|
||||
)
|
||||
return hidden_states, None
|
||||
return hidden_states
|
||||
|
||||
hidden_states_1 = self.post_feedforward_layernorm_1(hidden_states_1)
|
||||
hidden_states_2 = self.post_feedforward_layernorm_2(hidden_states_2)
|
||||
@@ -730,11 +784,9 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
# Combine branches
|
||||
hidden_states = hidden_states_1 + hidden_states_2
|
||||
else:
|
||||
# Fuse: hidden_states + residual -> residual; pre_ff_norm(residual) -> hidden_states
|
||||
hidden_states, residual = self.pre_feedforward_layernorm(
|
||||
hidden_states, residual
|
||||
hidden_states = self.ffn_boundary.sum_part(
|
||||
self.mlp(hidden_states), forward_batch, SumGroup.TP
|
||||
)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
|
||||
if (
|
||||
not self.has_ple
|
||||
@@ -766,7 +818,7 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
hidden_states = hidden_states + per_layer_contribution
|
||||
|
||||
hidden_states = hidden_states * self.layer_scalar
|
||||
return hidden_states, None
|
||||
return hidden_states
|
||||
|
||||
|
||||
class Gemma4TextModel(PreTrainedModel):
|
||||
@@ -977,49 +1029,45 @@ class Gemma4TextModel(PreTrainedModel):
|
||||
input_embeds, per_layer_inputs
|
||||
)
|
||||
hidden_states = input_embeds
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None, (
|
||||
"pp_proxy_tensors is required on non-first PP ranks"
|
||||
)
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
# PLE inputs were computed on rank 0 and forwarded along the
|
||||
# pipeline; non-PLE models simply omit the key.
|
||||
per_layer_inputs = pp_proxy_tensors.tensors.get("per_layer_inputs", None)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
num_layers = self.config.num_hidden_layers
|
||||
|
||||
for layer_idx in range(self.start_layer, self.end_layer):
|
||||
if layer_idx in self.layers_to_capture:
|
||||
aux_hidden_states.append(hidden_states)
|
||||
|
||||
if per_layer_inputs is not None:
|
||||
per_layer_input = per_layer_inputs[:, layer_idx, :]
|
||||
else:
|
||||
per_layer_input = None
|
||||
layer = self.layers[layer_idx]
|
||||
layer_outputs = layer(
|
||||
hidden_states = self.layers[layer_idx](
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
per_layer_input=per_layer_input,
|
||||
forward_batch=forward_batch,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if layer_idx in self.layers_to_capture
|
||||
else None,
|
||||
**kwargs,
|
||||
)
|
||||
hidden_states = layer_outputs[0]
|
||||
# Gemma4DecoderLayer.forward always returns (hidden_states, None);
|
||||
# the residual is fused inside the layer, so nothing to thread.
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
# cuda_graph_runner allocates a fixed PP-proxy schema of
|
||||
# {hidden_states, residual} and KeyErrors if a model omits a key.
|
||||
# Gemma4 fuses the residual inside each layer so we don't have a
|
||||
# standalone tensor to forward; emit a zero placeholder instead so
|
||||
# graph replay can still copy it. The receiving stage never reads
|
||||
# this key.
|
||||
proxy = {
|
||||
"hidden_states": hidden_states,
|
||||
"residual": torch.zeros_like(hidden_states),
|
||||
}
|
||||
# Each layer writes the whole stream at its FFN exit, so it goes as
|
||||
# hidden_states alone. cuda_graph_runner allocates a fixed PP-proxy
|
||||
# schema of {hidden_states, residual} and KeyErrors if a model
|
||||
# omits a key, so a zero placeholder fills the residual; the
|
||||
# receiving stage never reads it.
|
||||
proxy = residual_batch.to_pp(hidden_states, forward_batch).tensors
|
||||
proxy["residual"] = torch.zeros_like(hidden_states)
|
||||
if per_layer_inputs is not None:
|
||||
proxy["per_layer_inputs"] = per_layer_inputs
|
||||
return PPProxyTensors(proxy)
|
||||
@@ -1030,7 +1078,9 @@ class Gemma4TextModel(PreTrainedModel):
|
||||
if num_layers in self.layers_to_capture:
|
||||
aux_hidden_states.append(hidden_states)
|
||||
|
||||
hidden_states = self.norm(hidden_states)
|
||||
hidden_states = residual_batch.final_norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -9,6 +9,17 @@ from transformers import PretrainedConfig
|
||||
from triton.language.extra import libdevice
|
||||
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
UNFUSED_NORM_READOUT,
|
||||
UnfusedNormReadout,
|
||||
)
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
@@ -180,8 +191,10 @@ class IQuestQ1MoEBlock(nn.Module):
|
||||
layer_id: int,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
self.reduce_results = reduce_results
|
||||
self.hidden_size = config.hidden_size
|
||||
self.num_experts = config.num_experts
|
||||
self.top_k = config.num_experts_per_tok
|
||||
@@ -215,7 +228,8 @@ class IQuestQ1MoEBlock(nn.Module):
|
||||
router_logits, _ = self.gate(hidden_states.float())
|
||||
topk_output = self.topk(hidden_states, router_logits)
|
||||
output = self.experts(hidden_states, topk_output)
|
||||
output = post_experts_all_reduce(output)
|
||||
if self.reduce_results:
|
||||
output = post_experts_all_reduce(output)
|
||||
return output.reshape(original_shape)
|
||||
|
||||
|
||||
@@ -225,6 +239,7 @@ class IQuestQ1DenseMLP(nn.Module):
|
||||
config: PretrainedConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
@@ -239,6 +254,7 @@ class IQuestQ1DenseMLP(nn.Module):
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
)
|
||||
if config.hidden_act != "silu":
|
||||
@@ -261,6 +277,7 @@ class IQuestQ1Attention(nn.Module):
|
||||
layer_id: int,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
parallel = get_parallel()
|
||||
@@ -303,6 +320,7 @@ class IQuestQ1Attention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
self.q_norm = IQuestQ1RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
@@ -397,6 +415,25 @@ class IQuestQ1Attention(nn.Module):
|
||||
return output
|
||||
|
||||
|
||||
class _NormReplacesResidualRead(UnfusedNormReadout):
|
||||
"""After the update, the norm of the stream is both the stage's input and
|
||||
the new stream: IQuest-Q1 layers after the first add their attention
|
||||
output to the normalized input."""
|
||||
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
hidden_states, _ = super().read(
|
||||
residual, norm, quant_format, post_residual_addition
|
||||
)
|
||||
return hidden_states, hidden_states
|
||||
|
||||
|
||||
_NORM_REPLACES_RESIDUAL = _NormReplacesResidualRead()
|
||||
|
||||
|
||||
def _is_moe_layer(config: PretrainedConfig, layer_id: int) -> bool:
|
||||
return layer_id not in (config.mlp_only_layers or [])
|
||||
|
||||
|
||||
class IQuestQ1DecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -420,12 +457,14 @@ class IQuestQ1DecoderLayer(nn.Module):
|
||||
layer_id=layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
if layer_id in (config.mlp_only_layers or []):
|
||||
if not _is_moe_layer(config, layer_id):
|
||||
self.mlp = IQuestQ1DenseMLP(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
else:
|
||||
self.mlp = IQuestQ1MoEBlock(
|
||||
@@ -433,6 +472,7 @@ class IQuestQ1DecoderLayer(nn.Module):
|
||||
layer_id=layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
self.is_first_layer = layer_id == 0
|
||||
if self.is_first_layer:
|
||||
@@ -444,6 +484,43 @@ class IQuestQ1DecoderLayer(nn.Module):
|
||||
else:
|
||||
self.attn_out_scale = config.attn_out_scale
|
||||
self.ffn_out_scale = config.ffn_out_scale
|
||||
# Intentional: layer 0 adds the raw input and an FFN output norm, later layers
|
||||
# add the normalized input without one; the MTP draft follows layer 0.
|
||||
moe = _is_moe_layer(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=UNFUSED_NORM_READOUT
|
||||
if self.is_first_layer
|
||||
else _NORM_REPLACES_RESIDUAL,
|
||||
output_transform=OutputTransform(self._attn_output),
|
||||
),
|
||||
self.attention_norm,
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=moe,
|
||||
next_layer_sparse=_is_moe_layer(config, layer_id + 1),
|
||||
read=UNFUSED_NORM_READOUT,
|
||||
output_transform=OutputTransform(self._ffn_output),
|
||||
),
|
||||
self.feed_forward_norm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=_is_moe_layer(config, layer_id - 1), next_layer_sparse=moe
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def _attn_output(self, attn_output: torch.Tensor) -> torch.Tensor:
|
||||
return self.attn_out_norm(attn_output) * self.attn_out_scale
|
||||
|
||||
def _ffn_output(self, mlp_output: torch.Tensor) -> torch.Tensor:
|
||||
if self.is_first_layer:
|
||||
return self.ffn_out_norm(mlp_output) * self.ffn_out_scale
|
||||
return mlp_output * self.ffn_out_scale
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -451,23 +528,13 @@ class IQuestQ1DecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
# Intentional: layer 0 adds the raw input and an FFN output norm, later layers
|
||||
# add the normalized input without one; the MTP draft follows layer 0.
|
||||
if self.is_first_layer:
|
||||
norm_hidden_states = self.attention_norm(hidden_states)
|
||||
attn_output = self.self_attn(positions, norm_hidden_states, forward_batch)
|
||||
hidden_states = hidden_states + self.attn_out_norm(attn_output) * (
|
||||
self.attn_out_scale
|
||||
)
|
||||
mlp_output = self.mlp(self.feed_forward_norm(hidden_states))
|
||||
return hidden_states + self.ffn_out_norm(mlp_output) * self.ffn_out_scale
|
||||
norm_hidden_states = self.attention_norm(hidden_states)
|
||||
attn_output = self.self_attn(positions, norm_hidden_states, forward_batch)
|
||||
hidden_states = norm_hidden_states + self.attn_out_norm(attn_output) * (
|
||||
self.attn_out_scale
|
||||
)
|
||||
mlp_output = self.mlp(self.feed_forward_norm(hidden_states))
|
||||
return hidden_states + mlp_output * self.ffn_out_scale
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
attn_output = self.self_attn(positions, hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(attn_output, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
mlp_output = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(mlp_output)
|
||||
|
||||
|
||||
class IQuestQ1Model(nn.Module):
|
||||
@@ -507,9 +574,11 @@ class IQuestQ1Model(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual_batch.start(forward_batch)
|
||||
for layer in self.layers:
|
||||
hidden_states = layer(positions, hidden_states, forward_batch)
|
||||
return self.norm(hidden_states)
|
||||
hidden_states = residual_batch.fold(hidden_states, forward_batch)
|
||||
return self.norm(residual_batch.take_output(hidden_states, forward_batch))
|
||||
|
||||
|
||||
_FUSED_SHARD_LAYOUT = (
|
||||
|
||||
@@ -52,9 +52,13 @@ import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.configs.zaya import ZayaConfig
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -64,6 +68,7 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.topk import StandardTopKOutput
|
||||
from sglang.srt.layers.moe.utils import adds_replicated_output_to_partial
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
@@ -120,6 +125,72 @@ class ResidualScaling(nn.Module):
|
||||
return residual, hidden_states
|
||||
|
||||
|
||||
class _ResidualMergeRead:
|
||||
"""How a ZAYA1 stage reads its input: its ResidualScaling, when the config
|
||||
has one, on the incoming contribution and the fp32 residual, their fp32
|
||||
sum as the new residual, and the input norm of that sum in the model
|
||||
dtype. The producer's update must be a plain add."""
|
||||
|
||||
is_plain_norm = False
|
||||
reads_before_dp_gather = False
|
||||
|
||||
def __init__(self, res_scale: Optional[ResidualScaling]):
|
||||
self.res_scale = res_scale
|
||||
|
||||
def _merge(self, residual, hidden_states):
|
||||
if self.res_scale is not None:
|
||||
residual, hidden_states = self.res_scale(residual, hidden_states)
|
||||
if residual is None:
|
||||
return hidden_states.float()
|
||||
return residual.float() + hidden_states.float()
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
return self._merge(None, hidden_states)
|
||||
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
if quant_format or post_residual_addition is not None:
|
||||
raise NotImplementedError(
|
||||
f"a ZAYA1 read with {quant_format=} or a post-residual addition"
|
||||
)
|
||||
hidden_states = _apply_norm_with_fp32_residual(
|
||||
norm, residual, norm.weight.dtype
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
def update_and_read(
|
||||
self,
|
||||
update,
|
||||
hidden_states,
|
||||
residual,
|
||||
norm,
|
||||
quant_format="",
|
||||
post_residual_addition=None,
|
||||
):
|
||||
if not update.is_plain_add:
|
||||
raise NotImplementedError("a ZAYA1 read after a non-plain update")
|
||||
return self.read(
|
||||
self._merge(residual, hidden_states),
|
||||
norm,
|
||||
quant_format,
|
||||
post_residual_addition,
|
||||
)
|
||||
|
||||
|
||||
class _MergedNorm:
|
||||
"""The final norm, in the norm calling convention, read through the same
|
||||
merge as the layers."""
|
||||
|
||||
def __init__(self, read: _ResidualMergeRead, norm: nn.Module):
|
||||
self.read = read
|
||||
self.norm = norm
|
||||
|
||||
def __call__(self, hidden_states, residual=None):
|
||||
if residual is None:
|
||||
residual = self.read.init_residual(hidden_states)
|
||||
return self.read.read(residual, self.norm)[0]
|
||||
return self.read.update_and_read(PLAIN_ADD, hidden_states, residual, self.norm)
|
||||
|
||||
|
||||
def _apply_norm_with_fp32_residual(
|
||||
norm: nn.Module,
|
||||
residual: torch.Tensor,
|
||||
@@ -874,14 +945,14 @@ class ZayaAttention(nn.Module):
|
||||
prefix=add_prefix("qkv", prefix),
|
||||
)
|
||||
|
||||
# RowParallel o_proj: per-rank input is the rank's q heads, full
|
||||
# output is replicated via the end-of-forward all-reduce.
|
||||
# RowParallel o_proj: per-rank input is the rank's q heads; the next
|
||||
# stage's input completes the sum.
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.q_dim_full,
|
||||
self.hidden_size,
|
||||
bias=bool(getattr(config, "attention_bias", False)),
|
||||
input_is_parallel=True,
|
||||
reduce_results=True,
|
||||
reduce_results=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
@@ -1084,37 +1155,36 @@ def mod_premask_experts(
|
||||
indices: torch.Tensor,
|
||||
num_moe_experts: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Mask the (per-rank, pre-all-reduce) expert output for the MOD skip path.
|
||||
"""Mask the (per-rank, partial) expert output for the MOD skip path.
|
||||
|
||||
Returns ``(mod_mask, masked_experts)`` where ``mod_mask`` is ``1`` for
|
||||
tokens routed to a real expert and ``0`` for tokens routed to the skip
|
||||
slot (``indices == num_moe_experts``), and
|
||||
``masked_experts = mod_mask * experts_out``.
|
||||
|
||||
The masking is applied *before* the cross-rank all-reduce so the single
|
||||
reduction yields ``mask · sum_r(partial_r) = mask · experts_out_full``
|
||||
without the replicated ``mod_out`` term being summed ``tp_size`` times.
|
||||
Pairs with :func:`mod_blend`, which adds the skip-path term back after the
|
||||
reduce. Kept as a free function so the MOD math is unit-testable without a
|
||||
live ``torch.distributed`` group.
|
||||
The mask is replicated, so the TP sum of the masked partials is
|
||||
``mask · experts_out_full``. Pairs with :func:`mod_blend`, which adds the
|
||||
skip-path term. Kept as a free function so the MOD math is unit-testable
|
||||
without a live ``torch.distributed`` group.
|
||||
"""
|
||||
mod_mask = (indices != num_moe_experts).to(experts_out.dtype)
|
||||
return mod_mask, mod_mask * experts_out
|
||||
|
||||
|
||||
def mod_blend(
|
||||
masked_experts_reduced: torch.Tensor,
|
||||
masked_experts: torch.Tensor,
|
||||
mod_mask: torch.Tensor,
|
||||
mod_out: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Combine the already-all-reduced masked expert output with the skip path.
|
||||
"""Add the skip path, weighted by ``(1 - mod_mask)``, to the masked expert
|
||||
output.
|
||||
|
||||
``mod_out`` (the skip-expert residual, ``hidden_states * prob``) is
|
||||
replicated on every rank, so it is folded in here -- after the reduce of
|
||||
``masked_experts`` -- weighted by ``(1 - mod_mask)``. See
|
||||
:func:`mod_premask_experts`.
|
||||
replicated on every rank. While ``masked_experts`` is still a partial TP
|
||||
sum, only one rank may add it (:func:`adds_replicated_output_to_partial`),
|
||||
so that the sum counts it once. See :func:`mod_premask_experts`.
|
||||
"""
|
||||
return masked_experts_reduced + (1.0 - mod_mask) * mod_out
|
||||
return masked_experts + (1.0 - mod_mask) * mod_out
|
||||
|
||||
|
||||
class ZayaBlock(nn.Module):
|
||||
@@ -1201,25 +1271,21 @@ class ZayaBlock(nn.Module):
|
||||
|
||||
experts_out = self.experts(hidden_states, topk_out)
|
||||
# ``mod_out`` is computed identically on every TP rank (both
|
||||
# ``hidden_states`` and ``probs`` are replicated). Fold the skip
|
||||
# mask into the per-rank partial experts output *before*
|
||||
# all-reduce so the single reduction yields:
|
||||
# ``hidden_states`` and ``probs`` are replicated). The output is a
|
||||
# partial TP sum that the stage boundary completes, so only one
|
||||
# rank adds the skip path:
|
||||
# sum_r(mask · partial_r) + (1 - mask) · mod_out
|
||||
# = mask · experts_out_full + (1 - mask) · mod_out
|
||||
# without double-counting ``mod_out`` by tp_size. The two steps are
|
||||
# ``mod_premask_experts`` / ``mod_blend`` so the math is testable
|
||||
# without a live distributed group.
|
||||
# The two steps are ``mod_premask_experts`` / ``mod_blend`` so the
|
||||
# math is testable without a live distributed group.
|
||||
mod_out = hidden_states * probs
|
||||
mod_mask, masked_experts = mod_premask_experts(
|
||||
mod_mask, hidden_out = mod_premask_experts(
|
||||
experts_out, indices, self.num_moe_experts
|
||||
)
|
||||
if self.tp_size > 1:
|
||||
masked_experts = tensor_model_parallel_all_reduce(masked_experts)
|
||||
hidden_out = mod_blend(masked_experts, mod_mask, mod_out)
|
||||
if adds_replicated_output_to_partial():
|
||||
hidden_out = mod_blend(hidden_out, mod_mask, mod_out)
|
||||
else:
|
||||
hidden_out = self.experts(hidden_states, topk_out)
|
||||
if self.tp_size > 1:
|
||||
hidden_out = tensor_model_parallel_all_reduce(hidden_out)
|
||||
|
||||
return hidden_out, router_hs_next
|
||||
|
||||
@@ -1254,6 +1320,14 @@ class ZayaDecoderATTLayer(nn.Module):
|
||||
self.res_scale = ResidualScaling(config, layer_id)
|
||||
else:
|
||||
self.res_scale = None
|
||||
(self.attn_boundary,) = make_stages(
|
||||
(declare_attn(read=_ResidualMergeRead(self.res_scale)), self.input_norm),
|
||||
previous=declare_ffn(sparse=True, next_layer_sparse=True)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
self.entry_boundary = self.attn_boundary
|
||||
|
||||
@staticmethod
|
||||
def _build_norm(config: ZayaConfig) -> nn.Module:
|
||||
@@ -1266,27 +1340,14 @@ class ZayaDecoderATTLayer(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
prev_router_hidden_states: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
|
||||
target_dtype = (
|
||||
self.input_norm.weight.dtype
|
||||
if isinstance(self.input_norm, RMSNorm)
|
||||
else hidden_states.dtype
|
||||
)
|
||||
if self.res_scale is not None:
|
||||
residual, hidden_states = self.res_scale(residual, hidden_states)
|
||||
if residual is not None:
|
||||
residual = residual.float() + hidden_states.float()
|
||||
else:
|
||||
residual = hidden_states.float()
|
||||
hidden_states = _apply_norm_with_fp32_residual(
|
||||
self.input_norm, residual, target_dtype
|
||||
)
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.self_attn(hidden_states, positions, forward_batch)
|
||||
return hidden_states, residual, prev_router_hidden_states
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
return hidden_states, prev_router_hidden_states
|
||||
|
||||
|
||||
class ZayaDecoderMLPLayer(nn.Module):
|
||||
@@ -1314,33 +1375,34 @@ class ZayaDecoderMLPLayer(nn.Module):
|
||||
self.res_scale = ResidualScaling(config, layer_id)
|
||||
else:
|
||||
self.res_scale = None
|
||||
(self.ffn_boundary,) = make_stages(
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=True,
|
||||
next_layer_sparse=True,
|
||||
read=_ResidualMergeRead(self.res_scale),
|
||||
),
|
||||
self.input_norm,
|
||||
),
|
||||
previous=declare_attn(),
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
self.entry_boundary = self.ffn_boundary
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
prev_router_hidden_states: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
|
||||
target_dtype = (
|
||||
self.input_norm.weight.dtype
|
||||
if isinstance(self.input_norm, RMSNorm)
|
||||
else hidden_states.dtype
|
||||
)
|
||||
if self.res_scale is not None:
|
||||
residual, hidden_states = self.res_scale(residual, hidden_states)
|
||||
if residual is not None:
|
||||
residual = residual.float() + hidden_states.float()
|
||||
else:
|
||||
residual = hidden_states.float()
|
||||
hidden_states = _apply_norm_with_fp32_residual(
|
||||
self.input_norm, residual, target_dtype
|
||||
)
|
||||
hidden_states, prev_router_hidden_states = self.zaya_block(
|
||||
hidden_states, prev_router_hidden_states
|
||||
)
|
||||
return hidden_states, residual, prev_router_hidden_states
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states, prev_router_hidden_states = self.zaya_block(
|
||||
hidden_states, prev_router_hidden_states
|
||||
)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
return hidden_states, prev_router_hidden_states
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1412,6 +1474,9 @@ class ZayaModel(nn.Module):
|
||||
self.res_scale = ResidualScaling(config, config.num_hidden_layers)
|
||||
else:
|
||||
self.res_scale = None
|
||||
self._final_merged_norm = _MergedNorm(
|
||||
_ResidualMergeRead(self.res_scale), self.final_norm
|
||||
)
|
||||
else:
|
||||
self.final_norm = PPMissingLayer()
|
||||
self.res_scale = None
|
||||
@@ -1429,46 +1494,27 @@ class ZayaModel(nn.Module):
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
residual = None
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states = self.layers[self.start_layer].entry_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
prev_router_hidden_states: Optional[torch.Tensor] = None
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual, prev_router_hidden_states = layer(
|
||||
hidden_states, prev_router_hidden_states = self.layers[i](
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
prev_router_hidden_states=prev_router_hidden_states,
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
|
||||
if self.res_scale is not None:
|
||||
residual, hidden_states = self.res_scale(residual, hidden_states)
|
||||
target_dtype = (
|
||||
self.final_norm.weight.dtype
|
||||
if isinstance(self.final_norm, RMSNorm)
|
||||
else hidden_states.dtype
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
return residual_batch.final_norm(
|
||||
hidden_states, forward_batch, self._final_merged_norm
|
||||
)
|
||||
if residual is not None:
|
||||
merged = hidden_states.float() + residual.float()
|
||||
else:
|
||||
merged = hidden_states.float()
|
||||
hidden_states = _apply_norm_with_fp32_residual(
|
||||
self.final_norm, merged, target_dtype
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class ZayaForCausalLM(nn.Module):
|
||||
|
||||
Reference in New Issue
Block a user