[Refactor] Build the ZAYA1, IQuest-Q1 and Gemma 4 decoders from stage boundaries (#42311)

This commit is contained in:
Cheng Wan
2026-10-03 02:21:34 -07:00
committed by GitHub
parent f85c2c4923
commit 71b04e02cf
10 changed files with 433 additions and 174 deletions
@@ -46,6 +46,7 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import (
NORM_QUANT_READOUT,
NORM_READOUT,
PLAIN_ADD,
REPLACE_AT_EXIT,
)
from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
from sglang.srt.runtime_context import get_exec, get_parallel
@@ -156,9 +157,11 @@ def _resolve_ffn(
*((TokenAxis.ATTN_CP,) if on_cp_shards else ()), axis_sizes=axes
)
)
# An FFN that writes the next stream itself hands on a complete output.
complete = on_rank_rows or update is REPLACE_AT_EXIT
produced = (
OutputContract(rows, update=update, transform=output_transform)
if on_rank_rows
if complete
else OutputContract(
rows,
group=group,
@@ -280,6 +280,26 @@ class PlainAdd:
return attn_tp_gather(residual)
class ReplaceAtExit:
"""The producer computes the next stream itself (its own add, norms and
scaling, in whatever kernels it uses) and writes it at its exit; the
previous residual is dropped. The stream is complete: the producer has its
boundary complete each part's sum first (StageBoundary.sum_part)."""
is_plain_add = False
applied_at_exit = True
outlives_layer = True
def update(self, hidden_states, residual):
return hidden_states
def slice_residual_attn_tp(self, residual):
return attn_tp_slice(residual)
def gather_residual_attn_tp(self, residual):
return attn_tp_gather(residual)
class Fp8Input(Enum):
"""Additional input forms a projection can consume after a fused norm."""
@@ -390,6 +410,7 @@ class UnfusedNormReadout(NormReadout):
PLAIN_ADD = PlainAdd()
REPLACE_AT_EXIT = ReplaceAtExit()
NORM_QUANT_READOUT = NormQuantReadout()
NORM_READOUT = NormReadout()
UNFUSED_NORM_READOUT = UnfusedNormReadout()
@@ -35,6 +35,12 @@ def stream_of(forward_batch):
return stream
def written_residual(forward_batch):
"""The residual the last prepare wrote, for a stage whose compute reads it
too (e.g. to write the next stream at its exit). Borrowed: do not modify."""
return stream_of(forward_batch).residual
def complete_output(hidden_states, forward_batch):
"""Complete the contribution while retaining its pending residual update."""
return stream_of(forward_batch).complete(hidden_states)
@@ -30,10 +30,12 @@ from sglang.srt.layers.layer_boundary.contracts import (
StageKind,
StagePath,
)
from sglang.srt.layers.layer_boundary.ops import attn_tp_gather_input
from sglang.srt.layers.layer_boundary.layout import SumGroup
from sglang.srt.layers.layer_boundary.ops import attn_tp_gather_input, sum_output
from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp
from sglang.srt.layers.layer_boundary.residual.add_norm import (
PLAIN_ADD,
REPLACE_AT_EXIT,
)
from sglang.srt.layers.layer_boundary.residual.batch import stream_of
from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum, ResidualStream
@@ -262,6 +264,20 @@ class StageBoundary:
hidden_states, stream_of(forward_batch), forward_batch
)
def sum_part(self, hidden_states, forward_batch, group: SumGroup):
"""Complete the sum over ``group`` that one part of this FFN's output
owes.
For an FFN that builds the next stream itself (REPLACE_AT_EXIT) from
several complete parts, normalizing each before combining them; the
stream it hands to its exit is then complete.
"""
if self.declaration.update is not REPLACE_AT_EXIT:
raise RuntimeError(
"only an FFN that writes the next stream itself sums its parts"
)
return sum_output(hidden_states, group, forward_batch, may_quantize=False)
def branch_input(self, source, hidden_states, forward_batch):
hidden_states, stream = branch.branch_input(
self.plan,
@@ -83,6 +83,7 @@ class Gemma3MLP(nn.Module):
hidden_activation: str,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
reduce_results: bool = True,
) -> None:
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
@@ -97,6 +98,7 @@ class Gemma3MLP(nn.Module):
hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
)
if hidden_activation != "gelu_pytorch_tanh":
+96 -46
View File
@@ -33,6 +33,16 @@ from sglang.kernels.ops.moe.gemma4_routing import (
gemma4_fused_routing,
gemma_routing_post_topk,
)
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.layer_boundary import (
SumGroup,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.add_norm import REPLACE_AT_EXIT
from sglang.srt.layers.layernorm import Gemma4RMSNorm, RMSNorm
from sglang.srt.layers.linear import (
QKVParallelLinear,
@@ -219,6 +229,7 @@ class Gemma4MoE(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
activation: str = "gelu",
reduce_results: bool = True,
) -> None:
super().__init__()
self.layer_id = layer_id
@@ -277,7 +288,7 @@ class Gemma4MoE(nn.Module):
quant_config=quant_config,
prefix=add_prefix("experts", prefix),
activation=activation,
reduce_results=True,
reduce_results=reduce_results,
)
def forward(
@@ -352,6 +363,7 @@ class Gemma4Attention(nn.Module):
hidden_size,
bias=config.attention_bias,
quant_config=quant_config,
reduce_results=False,
prefix=add_prefix("o_proj", prefix),
)
@@ -578,6 +590,7 @@ class Gemma4DecoderLayer(nn.Module):
hidden_activation=config.hidden_activation,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -631,6 +644,7 @@ class Gemma4DecoderLayer(nn.Module):
config=config,
quant_config=quant_config,
prefix=add_prefix("moe", prefix),
reduce_results=False,
)
self.post_feedforward_layernorm_1 = RMSNorm(
@@ -652,6 +666,30 @@ class Gemma4DecoderLayer(nn.Module):
self.register_buffer("layer_scalar", torch.ones(1), persistent=True)
self.has_ple = self.hidden_size_per_layer_input > 0
self.prefix = prefix
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(
output_transform=OutputTransform(self.post_attention_layernorm)
),
self.input_layernorm,
),
(
declare_ffn(
sparse=self.enable_moe_block,
next_layer_sparse=self.enable_moe_block,
update=REPLACE_AT_EXIT,
),
self.pre_feedforward_layernorm,
),
previous=declare_ffn(
sparse=self.enable_moe_block,
next_layer_sparse=self.enable_moe_block,
update=REPLACE_AT_EXIT,
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
self,
@@ -659,47 +697,63 @@ class Gemma4DecoderLayer(nn.Module):
hidden_states: torch.Tensor,
per_layer_input: torch.Tensor,
forward_batch: ForwardBatch,
capture_output=None,
**kwargs,
) -> tuple[
torch.FloatTensor, Optional[tuple[torch.FloatTensor, torch.FloatTensor]]
]:
) -> torch.Tensor:
# Gemma4 residual pattern following JAX implementation:
# 1. input_norm(x) -> attn -> post_attn_norm -> ADD residual
# 2. pre_ff_norm -> mlp -> post_ff_norm -> ADD residual
#
# Optimization: fuse "post_attn_norm(h) + residual; pre_ff_norm(...)"
# into "post_attn_norm(h); pre_ff_norm(h, residual)" using
# gemma_fused_add_rmsnorm which computes:
# The FFN input runs post_attn_norm(h), then pre_ff_norm(h, residual)
# with gemma_fused_add_rmsnorm, which computes:
# residual = h + residual (in-place)
# h = gemma_norm(residual)
residual = hidden_states
# Apply input layernorm
hidden_states = self.input_layernorm(hidden_states)
hidden_states = self.attn_boundary.prepare(
hidden_states, forward_batch, capture=capture_output
)
hidden_states = self.self_attn(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
)
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
residual = residual_batch.written_residual(forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self._next_stream(
hidden_states, residual, per_layer_input, forward_batch
)
return ffn_exit.finish(hidden_states)
def _next_stream(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor,
per_layer_input: Optional[torch.Tensor],
forward_batch: ForwardBatch,
) -> torch.Tensor:
"""The next stream from the FFN input and the residual the FFN input
wrote. Each part's TP sum is completed by the FFN boundary before its
post-FFN norm."""
if self.enable_moe_block:
# Fuse: hidden_states + residual -> residual; pre_ff_norm(residual) -> hidden_states
# Also need raw (unfused) residual for router and pre_ff_norm_2
hidden_states, residual = self.pre_feedforward_layernorm(
hidden_states, residual
)
# For MoE: router and pre_ff_norm_2 need the unfused residual
# (which is now updated to post_attn_out + old_residual)
moe_input = residual
# Dense MLP branch
hidden_states_1 = self.mlp(hidden_states)
hidden_states_1 = self.ffn_boundary.sum_part(
self.mlp(hidden_states), forward_batch, SumGroup.TP
)
# MoE branch: router sees residual (= post_attn_out + old_residual)
router_logits = self.router(moe_input)
hidden_states_2 = self.pre_feedforward_layernorm_2(moe_input)
hidden_states_2 = self.moe(hidden_states_2, router_logits)
hidden_states_2 = self.ffn_boundary.sum_part(
self.moe(hidden_states_2, router_logits),
forward_batch,
SumGroup.MOE_OUTPUT,
)
# Fused: (rmsnorm(rmsnorm(h1,w1) + rmsnorm(h2,w2), w3) + residual) * scalar
if (
@@ -722,7 +776,7 @@ class Gemma4DecoderLayer(nn.Module):
norm2.variance_epsilon,
norm3.variance_epsilon,
)
return hidden_states, None
return hidden_states
hidden_states_1 = self.post_feedforward_layernorm_1(hidden_states_1)
hidden_states_2 = self.post_feedforward_layernorm_2(hidden_states_2)
@@ -730,11 +784,9 @@ class Gemma4DecoderLayer(nn.Module):
# Combine branches
hidden_states = hidden_states_1 + hidden_states_2
else:
# Fuse: hidden_states + residual -> residual; pre_ff_norm(residual) -> hidden_states
hidden_states, residual = self.pre_feedforward_layernorm(
hidden_states, residual
hidden_states = self.ffn_boundary.sum_part(
self.mlp(hidden_states), forward_batch, SumGroup.TP
)
hidden_states = self.mlp(hidden_states)
if (
not self.has_ple
@@ -766,7 +818,7 @@ class Gemma4DecoderLayer(nn.Module):
hidden_states = hidden_states + per_layer_contribution
hidden_states = hidden_states * self.layer_scalar
return hidden_states, None
return hidden_states
class Gemma4TextModel(PreTrainedModel):
@@ -977,49 +1029,45 @@ class Gemma4TextModel(PreTrainedModel):
input_embeds, per_layer_inputs
)
hidden_states = input_embeds
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None, (
"pp_proxy_tensors is required on non-first PP ranks"
)
hidden_states = pp_proxy_tensors["hidden_states"]
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
# PLE inputs were computed on rank 0 and forwarded along the
# pipeline; non-PLE models simply omit the key.
per_layer_inputs = pp_proxy_tensors.tensors.get("per_layer_inputs", None)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
num_layers = self.config.num_hidden_layers
for layer_idx in range(self.start_layer, self.end_layer):
if layer_idx in self.layers_to_capture:
aux_hidden_states.append(hidden_states)
if per_layer_inputs is not None:
per_layer_input = per_layer_inputs[:, layer_idx, :]
else:
per_layer_input = None
layer = self.layers[layer_idx]
layer_outputs = layer(
hidden_states = self.layers[layer_idx](
positions=positions,
hidden_states=hidden_states,
per_layer_input=per_layer_input,
forward_batch=forward_batch,
capture_output=aux_hidden_states.capture
if layer_idx in self.layers_to_capture
else None,
**kwargs,
)
hidden_states = layer_outputs[0]
# Gemma4DecoderLayer.forward always returns (hidden_states, None);
# the residual is fused inside the layer, so nothing to thread.
if not self.pp_group.is_last_rank:
# cuda_graph_runner allocates a fixed PP-proxy schema of
# {hidden_states, residual} and KeyErrors if a model omits a key.
# Gemma4 fuses the residual inside each layer so we don't have a
# standalone tensor to forward; emit a zero placeholder instead so
# graph replay can still copy it. The receiving stage never reads
# this key.
proxy = {
"hidden_states": hidden_states,
"residual": torch.zeros_like(hidden_states),
}
# Each layer writes the whole stream at its FFN exit, so it goes as
# hidden_states alone. cuda_graph_runner allocates a fixed PP-proxy
# schema of {hidden_states, residual} and KeyErrors if a model
# omits a key, so a zero placeholder fills the residual; the
# receiving stage never reads it.
proxy = residual_batch.to_pp(hidden_states, forward_batch).tensors
proxy["residual"] = torch.zeros_like(hidden_states)
if per_layer_inputs is not None:
proxy["per_layer_inputs"] = per_layer_inputs
return PPProxyTensors(proxy)
@@ -1030,7 +1078,9 @@ class Gemma4TextModel(PreTrainedModel):
if num_layers in self.layers_to_capture:
aux_hidden_states.append(hidden_states)
hidden_states = self.norm(hidden_states)
hidden_states = residual_batch.final_norm(
hidden_states, forward_batch, self.norm
)
if len(aux_hidden_states) == 0:
return hidden_states
+89 -20
View File
@@ -9,6 +9,17 @@ from transformers import PretrainedConfig
from triton.language.extra import libdevice
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.add_norm import (
UNFUSED_NORM_READOUT,
UnfusedNormReadout,
)
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
@@ -180,8 +191,10 @@ class IQuestQ1MoEBlock(nn.Module):
layer_id: int,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
reduce_results: bool = True,
):
super().__init__()
self.reduce_results = reduce_results
self.hidden_size = config.hidden_size
self.num_experts = config.num_experts
self.top_k = config.num_experts_per_tok
@@ -215,7 +228,8 @@ class IQuestQ1MoEBlock(nn.Module):
router_logits, _ = self.gate(hidden_states.float())
topk_output = self.topk(hidden_states, router_logits)
output = self.experts(hidden_states, topk_output)
output = post_experts_all_reduce(output)
if self.reduce_results:
output = post_experts_all_reduce(output)
return output.reshape(original_shape)
@@ -225,6 +239,7 @@ class IQuestQ1DenseMLP(nn.Module):
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
reduce_results: bool = True,
):
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
@@ -239,6 +254,7 @@ class IQuestQ1DenseMLP(nn.Module):
config.hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
)
if config.hidden_act != "silu":
@@ -261,6 +277,7 @@ class IQuestQ1Attention(nn.Module):
layer_id: int,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
reduce_results: bool = True,
):
super().__init__()
parallel = get_parallel()
@@ -303,6 +320,7 @@ class IQuestQ1Attention(nn.Module):
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
reduce_results=reduce_results,
prefix=add_prefix("o_proj", prefix),
)
self.q_norm = IQuestQ1RMSNorm(self.head_dim, eps=config.rms_norm_eps)
@@ -397,6 +415,25 @@ class IQuestQ1Attention(nn.Module):
return output
class _NormReplacesResidualRead(UnfusedNormReadout):
"""After the update, the norm of the stream is both the stage's input and
the new stream: IQuest-Q1 layers after the first add their attention
output to the normalized input."""
def read(self, residual, norm, quant_format="", post_residual_addition=None):
hidden_states, _ = super().read(
residual, norm, quant_format, post_residual_addition
)
return hidden_states, hidden_states
_NORM_REPLACES_RESIDUAL = _NormReplacesResidualRead()
def _is_moe_layer(config: PretrainedConfig, layer_id: int) -> bool:
return layer_id not in (config.mlp_only_layers or [])
class IQuestQ1DecoderLayer(nn.Module):
def __init__(
self,
@@ -420,12 +457,14 @@ class IQuestQ1DecoderLayer(nn.Module):
layer_id=layer_id,
quant_config=quant_config,
prefix=add_prefix("self_attn", prefix),
reduce_results=False,
)
if layer_id in (config.mlp_only_layers or []):
if not _is_moe_layer(config, layer_id):
self.mlp = IQuestQ1DenseMLP(
config=config,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
else:
self.mlp = IQuestQ1MoEBlock(
@@ -433,6 +472,7 @@ class IQuestQ1DecoderLayer(nn.Module):
layer_id=layer_id,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
self.is_first_layer = layer_id == 0
if self.is_first_layer:
@@ -444,6 +484,43 @@ class IQuestQ1DecoderLayer(nn.Module):
else:
self.attn_out_scale = config.attn_out_scale
self.ffn_out_scale = config.ffn_out_scale
# Intentional: layer 0 adds the raw input and an FFN output norm, later layers
# add the normalized input without one; the MTP draft follows layer 0.
moe = _is_moe_layer(config, layer_id)
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(
read=UNFUSED_NORM_READOUT
if self.is_first_layer
else _NORM_REPLACES_RESIDUAL,
output_transform=OutputTransform(self._attn_output),
),
self.attention_norm,
),
(
declare_ffn(
sparse=moe,
next_layer_sparse=_is_moe_layer(config, layer_id + 1),
read=UNFUSED_NORM_READOUT,
output_transform=OutputTransform(self._ffn_output),
),
self.feed_forward_norm,
),
previous=declare_ffn(
sparse=_is_moe_layer(config, layer_id - 1), next_layer_sparse=moe
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def _attn_output(self, attn_output: torch.Tensor) -> torch.Tensor:
return self.attn_out_norm(attn_output) * self.attn_out_scale
def _ffn_output(self, mlp_output: torch.Tensor) -> torch.Tensor:
if self.is_first_layer:
return self.ffn_out_norm(mlp_output) * self.ffn_out_scale
return mlp_output * self.ffn_out_scale
def forward(
self,
@@ -451,23 +528,13 @@ class IQuestQ1DecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> torch.Tensor:
# Intentional: layer 0 adds the raw input and an FFN output norm, later layers
# add the normalized input without one; the MTP draft follows layer 0.
if self.is_first_layer:
norm_hidden_states = self.attention_norm(hidden_states)
attn_output = self.self_attn(positions, norm_hidden_states, forward_batch)
hidden_states = hidden_states + self.attn_out_norm(attn_output) * (
self.attn_out_scale
)
mlp_output = self.mlp(self.feed_forward_norm(hidden_states))
return hidden_states + self.ffn_out_norm(mlp_output) * self.ffn_out_scale
norm_hidden_states = self.attention_norm(hidden_states)
attn_output = self.self_attn(positions, norm_hidden_states, forward_batch)
hidden_states = norm_hidden_states + self.attn_out_norm(attn_output) * (
self.attn_out_scale
)
mlp_output = self.mlp(self.feed_forward_norm(hidden_states))
return hidden_states + mlp_output * self.ffn_out_scale
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
attn_output = self.self_attn(positions, hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(attn_output, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
mlp_output = self.mlp(hidden_states)
return ffn_exit.finish(mlp_output)
class IQuestQ1Model(nn.Module):
@@ -507,9 +574,11 @@ class IQuestQ1Model(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual_batch.start(forward_batch)
for layer in self.layers:
hidden_states = layer(positions, hidden_states, forward_batch)
return self.norm(hidden_states)
hidden_states = residual_batch.fold(hidden_states, forward_batch)
return self.norm(residual_batch.take_output(hidden_states, forward_batch))
_FUSED_SHARD_LAYOUT = (
+140 -94
View File
@@ -52,9 +52,13 @@ import torch.nn.functional as F
from torch import nn
from sglang.srt.configs.zaya import ZayaConfig
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -64,6 +68,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.topk import StandardTopKOutput
from sglang.srt.layers.moe.utils import adds_replicated_output_to_partial
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
@@ -120,6 +125,72 @@ class ResidualScaling(nn.Module):
return residual, hidden_states
class _ResidualMergeRead:
"""How a ZAYA1 stage reads its input: its ResidualScaling, when the config
has one, on the incoming contribution and the fp32 residual, their fp32
sum as the new residual, and the input norm of that sum in the model
dtype. The producer's update must be a plain add."""
is_plain_norm = False
reads_before_dp_gather = False
def __init__(self, res_scale: Optional[ResidualScaling]):
self.res_scale = res_scale
def _merge(self, residual, hidden_states):
if self.res_scale is not None:
residual, hidden_states = self.res_scale(residual, hidden_states)
if residual is None:
return hidden_states.float()
return residual.float() + hidden_states.float()
def init_residual(self, hidden_states):
return self._merge(None, hidden_states)
def read(self, residual, norm, quant_format="", post_residual_addition=None):
if quant_format or post_residual_addition is not None:
raise NotImplementedError(
f"a ZAYA1 read with {quant_format=} or a post-residual addition"
)
hidden_states = _apply_norm_with_fp32_residual(
norm, residual, norm.weight.dtype
)
return hidden_states, residual
def update_and_read(
self,
update,
hidden_states,
residual,
norm,
quant_format="",
post_residual_addition=None,
):
if not update.is_plain_add:
raise NotImplementedError("a ZAYA1 read after a non-plain update")
return self.read(
self._merge(residual, hidden_states),
norm,
quant_format,
post_residual_addition,
)
class _MergedNorm:
"""The final norm, in the norm calling convention, read through the same
merge as the layers."""
def __init__(self, read: _ResidualMergeRead, norm: nn.Module):
self.read = read
self.norm = norm
def __call__(self, hidden_states, residual=None):
if residual is None:
residual = self.read.init_residual(hidden_states)
return self.read.read(residual, self.norm)[0]
return self.read.update_and_read(PLAIN_ADD, hidden_states, residual, self.norm)
def _apply_norm_with_fp32_residual(
norm: nn.Module,
residual: torch.Tensor,
@@ -874,14 +945,14 @@ class ZayaAttention(nn.Module):
prefix=add_prefix("qkv", prefix),
)
# RowParallel o_proj: per-rank input is the rank's q heads, full
# output is replicated via the end-of-forward all-reduce.
# RowParallel o_proj: per-rank input is the rank's q heads; the next
# stage's input completes the sum.
self.o_proj = RowParallelLinear(
self.q_dim_full,
self.hidden_size,
bias=bool(getattr(config, "attention_bias", False)),
input_is_parallel=True,
reduce_results=True,
reduce_results=False,
quant_config=quant_config,
prefix=add_prefix("o_proj", prefix),
)
@@ -1084,37 +1155,36 @@ def mod_premask_experts(
indices: torch.Tensor,
num_moe_experts: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Mask the (per-rank, pre-all-reduce) expert output for the MOD skip path.
"""Mask the (per-rank, partial) expert output for the MOD skip path.
Returns ``(mod_mask, masked_experts)`` where ``mod_mask`` is ``1`` for
tokens routed to a real expert and ``0`` for tokens routed to the skip
slot (``indices == num_moe_experts``), and
``masked_experts = mod_mask * experts_out``.
The masking is applied *before* the cross-rank all-reduce so the single
reduction yields ``mask · sum_r(partial_r) = mask · experts_out_full``
without the replicated ``mod_out`` term being summed ``tp_size`` times.
Pairs with :func:`mod_blend`, which adds the skip-path term back after the
reduce. Kept as a free function so the MOD math is unit-testable without a
live ``torch.distributed`` group.
The mask is replicated, so the TP sum of the masked partials is
``mask · experts_out_full``. Pairs with :func:`mod_blend`, which adds the
skip-path term. Kept as a free function so the MOD math is unit-testable
without a live ``torch.distributed`` group.
"""
mod_mask = (indices != num_moe_experts).to(experts_out.dtype)
return mod_mask, mod_mask * experts_out
def mod_blend(
masked_experts_reduced: torch.Tensor,
masked_experts: torch.Tensor,
mod_mask: torch.Tensor,
mod_out: torch.Tensor,
) -> torch.Tensor:
"""Combine the already-all-reduced masked expert output with the skip path.
"""Add the skip path, weighted by ``(1 - mod_mask)``, to the masked expert
output.
``mod_out`` (the skip-expert residual, ``hidden_states * prob``) is
replicated on every rank, so it is folded in here -- after the reduce of
``masked_experts`` -- weighted by ``(1 - mod_mask)``. See
:func:`mod_premask_experts`.
replicated on every rank. While ``masked_experts`` is still a partial TP
sum, only one rank may add it (:func:`adds_replicated_output_to_partial`),
so that the sum counts it once. See :func:`mod_premask_experts`.
"""
return masked_experts_reduced + (1.0 - mod_mask) * mod_out
return masked_experts + (1.0 - mod_mask) * mod_out
class ZayaBlock(nn.Module):
@@ -1201,25 +1271,21 @@ class ZayaBlock(nn.Module):
experts_out = self.experts(hidden_states, topk_out)
# ``mod_out`` is computed identically on every TP rank (both
# ``hidden_states`` and ``probs`` are replicated). Fold the skip
# mask into the per-rank partial experts output *before*
# all-reduce so the single reduction yields:
# ``hidden_states`` and ``probs`` are replicated). The output is a
# partial TP sum that the stage boundary completes, so only one
# rank adds the skip path:
# sum_r(mask · partial_r) + (1 - mask) · mod_out
# = mask · experts_out_full + (1 - mask) · mod_out
# without double-counting ``mod_out`` by tp_size. The two steps are
# ``mod_premask_experts`` / ``mod_blend`` so the math is testable
# without a live distributed group.
# The two steps are ``mod_premask_experts`` / ``mod_blend`` so the
# math is testable without a live distributed group.
mod_out = hidden_states * probs
mod_mask, masked_experts = mod_premask_experts(
mod_mask, hidden_out = mod_premask_experts(
experts_out, indices, self.num_moe_experts
)
if self.tp_size > 1:
masked_experts = tensor_model_parallel_all_reduce(masked_experts)
hidden_out = mod_blend(masked_experts, mod_mask, mod_out)
if adds_replicated_output_to_partial():
hidden_out = mod_blend(hidden_out, mod_mask, mod_out)
else:
hidden_out = self.experts(hidden_states, topk_out)
if self.tp_size > 1:
hidden_out = tensor_model_parallel_all_reduce(hidden_out)
return hidden_out, router_hs_next
@@ -1254,6 +1320,14 @@ class ZayaDecoderATTLayer(nn.Module):
self.res_scale = ResidualScaling(config, layer_id)
else:
self.res_scale = None
(self.attn_boundary,) = make_stages(
(declare_attn(read=_ResidualMergeRead(self.res_scale)), self.input_norm),
previous=declare_ffn(sparse=True, next_layer_sparse=True)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
self.entry_boundary = self.attn_boundary
@staticmethod
def _build_norm(config: ZayaConfig) -> nn.Module:
@@ -1266,27 +1340,14 @@ class ZayaDecoderATTLayer(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
positions: torch.Tensor,
forward_batch: ForwardBatch,
prev_router_hidden_states: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
target_dtype = (
self.input_norm.weight.dtype
if isinstance(self.input_norm, RMSNorm)
else hidden_states.dtype
)
if self.res_scale is not None:
residual, hidden_states = self.res_scale(residual, hidden_states)
if residual is not None:
residual = residual.float() + hidden_states.float()
else:
residual = hidden_states.float()
hidden_states = _apply_norm_with_fp32_residual(
self.input_norm, residual, target_dtype
)
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.self_attn(hidden_states, positions, forward_batch)
return hidden_states, residual, prev_router_hidden_states
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
return hidden_states, prev_router_hidden_states
class ZayaDecoderMLPLayer(nn.Module):
@@ -1314,33 +1375,34 @@ class ZayaDecoderMLPLayer(nn.Module):
self.res_scale = ResidualScaling(config, layer_id)
else:
self.res_scale = None
(self.ffn_boundary,) = make_stages(
(
declare_ffn(
sparse=True,
next_layer_sparse=True,
read=_ResidualMergeRead(self.res_scale),
),
self.input_norm,
),
previous=declare_attn(),
terminal=layer_id == config.num_hidden_layers - 1,
)
self.entry_boundary = self.ffn_boundary
def forward(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
positions: torch.Tensor,
forward_batch: ForwardBatch,
prev_router_hidden_states: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
target_dtype = (
self.input_norm.weight.dtype
if isinstance(self.input_norm, RMSNorm)
else hidden_states.dtype
)
if self.res_scale is not None:
residual, hidden_states = self.res_scale(residual, hidden_states)
if residual is not None:
residual = residual.float() + hidden_states.float()
else:
residual = hidden_states.float()
hidden_states = _apply_norm_with_fp32_residual(
self.input_norm, residual, target_dtype
)
hidden_states, prev_router_hidden_states = self.zaya_block(
hidden_states, prev_router_hidden_states
)
return hidden_states, residual, prev_router_hidden_states
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states, prev_router_hidden_states = self.zaya_block(
hidden_states, prev_router_hidden_states
)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, prev_router_hidden_states
# ---------------------------------------------------------------------------
@@ -1412,6 +1474,9 @@ class ZayaModel(nn.Module):
self.res_scale = ResidualScaling(config, config.num_hidden_layers)
else:
self.res_scale = None
self._final_merged_norm = _MergedNorm(
_ResidualMergeRead(self.res_scale), self.final_norm
)
else:
self.final_norm = PPMissingLayer()
self.res_scale = None
@@ -1429,46 +1494,27 @@ class ZayaModel(nn.Module):
hidden_states = inputs_embeds
else:
hidden_states = self.embed_tokens(input_ids)
residual = None
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states = self.layers[self.start_layer].entry_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
prev_router_hidden_states: Optional[torch.Tensor] = None
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
hidden_states, residual, prev_router_hidden_states = layer(
hidden_states, prev_router_hidden_states = self.layers[i](
hidden_states=hidden_states,
residual=residual,
positions=positions,
forward_batch=forward_batch,
prev_router_hidden_states=prev_router_hidden_states,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
if self.res_scale is not None:
residual, hidden_states = self.res_scale(residual, hidden_states)
target_dtype = (
self.final_norm.weight.dtype
if isinstance(self.final_norm, RMSNorm)
else hidden_states.dtype
return residual_batch.to_pp(hidden_states, forward_batch)
return residual_batch.final_norm(
hidden_states, forward_batch, self._final_merged_norm
)
if residual is not None:
merged = hidden_states.float() + residual.float()
else:
merged = hidden_states.float()
hidden_states = _apply_norm_with_fp32_residual(
self.final_norm, merged, target_dtype
)
return hidden_states
class ZayaForCausalLM(nn.Module):