From 71b04e02cfd8cf04353c183fd71a22efc9b91722 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:21:34 -0700 Subject: [PATCH] [Refactor] Build the ZAYA1, IQuest-Q1 and Gemma 4 decoders from stage boundaries (#42311) --- .../srt/layers/layer_boundary/factories.py | 5 +- .../layer_boundary/residual/add_norm.py | 21 ++ .../layers/layer_boundary/residual/batch.py | 6 + .../sglang/srt/layers/layer_boundary/stage.py | 18 +- python/sglang/srt/models/gemma3_causal.py | 2 + python/sglang/srt/models/gemma4_causal.py | 142 +++++++---- python/sglang/srt/models/iquest_q1.py | 109 ++++++-- python/sglang/srt/models/zaya.py | 234 +++++++++++------- .../test_boundary_output_contracts.py | 45 ++++ .../unit/models/test_zaya_mod_tp.py | 25 +- 10 files changed, 433 insertions(+), 174 deletions(-) diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index 1e0d3f2fc2..816ca5fd5e 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -46,6 +46,7 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import ( NORM_QUANT_READOUT, NORM_READOUT, PLAIN_ADD, + REPLACE_AT_EXIT, ) from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks from sglang.srt.runtime_context import get_exec, get_parallel @@ -156,9 +157,11 @@ def _resolve_ffn( *((TokenAxis.ATTN_CP,) if on_cp_shards else ()), axis_sizes=axes ) ) + # An FFN that writes the next stream itself hands on a complete output. + complete = on_rank_rows or update is REPLACE_AT_EXIT produced = ( OutputContract(rows, update=update, transform=output_transform) - if on_rank_rows + if complete else OutputContract( rows, group=group, diff --git a/python/sglang/srt/layers/layer_boundary/residual/add_norm.py b/python/sglang/srt/layers/layer_boundary/residual/add_norm.py index 9f277c3721..6c732e7e85 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/add_norm.py +++ b/python/sglang/srt/layers/layer_boundary/residual/add_norm.py @@ -280,6 +280,26 @@ class PlainAdd: return attn_tp_gather(residual) +class ReplaceAtExit: + """The producer computes the next stream itself (its own add, norms and + scaling, in whatever kernels it uses) and writes it at its exit; the + previous residual is dropped. The stream is complete: the producer has its + boundary complete each part's sum first (StageBoundary.sum_part).""" + + is_plain_add = False + applied_at_exit = True + outlives_layer = True + + def update(self, hidden_states, residual): + return hidden_states + + def slice_residual_attn_tp(self, residual): + return attn_tp_slice(residual) + + def gather_residual_attn_tp(self, residual): + return attn_tp_gather(residual) + + class Fp8Input(Enum): """Additional input forms a projection can consume after a fused norm.""" @@ -390,6 +410,7 @@ class UnfusedNormReadout(NormReadout): PLAIN_ADD = PlainAdd() +REPLACE_AT_EXIT = ReplaceAtExit() NORM_QUANT_READOUT = NormQuantReadout() NORM_READOUT = NormReadout() UNFUSED_NORM_READOUT = UnfusedNormReadout() diff --git a/python/sglang/srt/layers/layer_boundary/residual/batch.py b/python/sglang/srt/layers/layer_boundary/residual/batch.py index 8ebb21e505..1d9b31a59d 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/batch.py +++ b/python/sglang/srt/layers/layer_boundary/residual/batch.py @@ -35,6 +35,12 @@ def stream_of(forward_batch): return stream +def written_residual(forward_batch): + """The residual the last prepare wrote, for a stage whose compute reads it + too (e.g. to write the next stream at its exit). Borrowed: do not modify.""" + return stream_of(forward_batch).residual + + def complete_output(hidden_states, forward_batch): """Complete the contribution while retaining its pending residual update.""" return stream_of(forward_batch).complete(hidden_states) diff --git a/python/sglang/srt/layers/layer_boundary/stage.py b/python/sglang/srt/layers/layer_boundary/stage.py index 06cbb8a125..e8af2e361e 100644 --- a/python/sglang/srt/layers/layer_boundary/stage.py +++ b/python/sglang/srt/layers/layer_boundary/stage.py @@ -30,10 +30,12 @@ from sglang.srt.layers.layer_boundary.contracts import ( StageKind, StagePath, ) -from sglang.srt.layers.layer_boundary.ops import attn_tp_gather_input +from sglang.srt.layers.layer_boundary.layout import SumGroup +from sglang.srt.layers.layer_boundary.ops import attn_tp_gather_input, sum_output from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp from sglang.srt.layers.layer_boundary.residual.add_norm import ( PLAIN_ADD, + REPLACE_AT_EXIT, ) from sglang.srt.layers.layer_boundary.residual.batch import stream_of from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum, ResidualStream @@ -262,6 +264,20 @@ class StageBoundary: hidden_states, stream_of(forward_batch), forward_batch ) + def sum_part(self, hidden_states, forward_batch, group: SumGroup): + """Complete the sum over ``group`` that one part of this FFN's output + owes. + + For an FFN that builds the next stream itself (REPLACE_AT_EXIT) from + several complete parts, normalizing each before combining them; the + stream it hands to its exit is then complete. + """ + if self.declaration.update is not REPLACE_AT_EXIT: + raise RuntimeError( + "only an FFN that writes the next stream itself sums its parts" + ) + return sum_output(hidden_states, group, forward_batch, may_quantize=False) + def branch_input(self, source, hidden_states, forward_batch): hidden_states, stream = branch.branch_input( self.plan, diff --git a/python/sglang/srt/models/gemma3_causal.py b/python/sglang/srt/models/gemma3_causal.py index 73f7409a5e..210156d84e 100644 --- a/python/sglang/srt/models/gemma3_causal.py +++ b/python/sglang/srt/models/gemma3_causal.py @@ -83,6 +83,7 @@ class Gemma3MLP(nn.Module): hidden_activation: str, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", + reduce_results: bool = True, ) -> None: super().__init__() self.gate_up_proj = MergedColumnParallelLinear( @@ -97,6 +98,7 @@ class Gemma3MLP(nn.Module): hidden_size, bias=False, quant_config=quant_config, + reduce_results=reduce_results, prefix=add_prefix("down_proj", prefix), ) if hidden_activation != "gelu_pytorch_tanh": diff --git a/python/sglang/srt/models/gemma4_causal.py b/python/sglang/srt/models/gemma4_causal.py index 65edfc658d..1cedbaec1d 100644 --- a/python/sglang/srt/models/gemma4_causal.py +++ b/python/sglang/srt/models/gemma4_causal.py @@ -33,6 +33,16 @@ from sglang.kernels.ops.moe.gemma4_routing import ( gemma4_fused_routing, gemma_routing_post_topk, ) +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList +from sglang.srt.layers.layer_boundary import ( + SumGroup, + declare_attn, + declare_ffn, + make_stages, +) +from sglang.srt.layers.layer_boundary.output import OutputTransform +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual.add_norm import REPLACE_AT_EXIT from sglang.srt.layers.layernorm import Gemma4RMSNorm, RMSNorm from sglang.srt.layers.linear import ( QKVParallelLinear, @@ -219,6 +229,7 @@ class Gemma4MoE(nn.Module): quant_config: Optional[QuantizationConfig] = None, prefix: str = "", activation: str = "gelu", + reduce_results: bool = True, ) -> None: super().__init__() self.layer_id = layer_id @@ -277,7 +288,7 @@ class Gemma4MoE(nn.Module): quant_config=quant_config, prefix=add_prefix("experts", prefix), activation=activation, - reduce_results=True, + reduce_results=reduce_results, ) def forward( @@ -352,6 +363,7 @@ class Gemma4Attention(nn.Module): hidden_size, bias=config.attention_bias, quant_config=quant_config, + reduce_results=False, prefix=add_prefix("o_proj", prefix), ) @@ -578,6 +590,7 @@ class Gemma4DecoderLayer(nn.Module): hidden_activation=config.hidden_activation, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -631,6 +644,7 @@ class Gemma4DecoderLayer(nn.Module): config=config, quant_config=quant_config, prefix=add_prefix("moe", prefix), + reduce_results=False, ) self.post_feedforward_layernorm_1 = RMSNorm( @@ -652,6 +666,30 @@ class Gemma4DecoderLayer(nn.Module): self.register_buffer("layer_scalar", torch.ones(1), persistent=True) self.has_ple = self.hidden_size_per_layer_input > 0 self.prefix = prefix + self.attn_boundary, self.ffn_boundary = make_stages( + ( + declare_attn( + output_transform=OutputTransform(self.post_attention_layernorm) + ), + self.input_layernorm, + ), + ( + declare_ffn( + sparse=self.enable_moe_block, + next_layer_sparse=self.enable_moe_block, + update=REPLACE_AT_EXIT, + ), + self.pre_feedforward_layernorm, + ), + previous=declare_ffn( + sparse=self.enable_moe_block, + next_layer_sparse=self.enable_moe_block, + update=REPLACE_AT_EXIT, + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, + ) def forward( self, @@ -659,47 +697,63 @@ class Gemma4DecoderLayer(nn.Module): hidden_states: torch.Tensor, per_layer_input: torch.Tensor, forward_batch: ForwardBatch, + capture_output=None, **kwargs, - ) -> tuple[ - torch.FloatTensor, Optional[tuple[torch.FloatTensor, torch.FloatTensor]] - ]: + ) -> torch.Tensor: # Gemma4 residual pattern following JAX implementation: # 1. input_norm(x) -> attn -> post_attn_norm -> ADD residual # 2. pre_ff_norm -> mlp -> post_ff_norm -> ADD residual # - # Optimization: fuse "post_attn_norm(h) + residual; pre_ff_norm(...)" - # into "post_attn_norm(h); pre_ff_norm(h, residual)" using - # gemma_fused_add_rmsnorm which computes: + # The FFN input runs post_attn_norm(h), then pre_ff_norm(h, residual) + # with gemma_fused_add_rmsnorm, which computes: # residual = h + residual (in-place) # h = gemma_norm(residual) - residual = hidden_states - - # Apply input layernorm - hidden_states = self.input_layernorm(hidden_states) + hidden_states = self.attn_boundary.prepare( + hidden_states, forward_batch, capture=capture_output + ) hidden_states = self.self_attn( positions=positions, hidden_states=hidden_states, forward_batch=forward_batch, ) - hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + residual = residual_batch.written_residual(forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + hidden_states = self._next_stream( + hidden_states, residual, per_layer_input, forward_batch + ) + return ffn_exit.finish(hidden_states) + + def _next_stream( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + per_layer_input: Optional[torch.Tensor], + forward_batch: ForwardBatch, + ) -> torch.Tensor: + """The next stream from the FFN input and the residual the FFN input + wrote. Each part's TP sum is completed by the FFN boundary before its + post-FFN norm.""" if self.enable_moe_block: - # Fuse: hidden_states + residual -> residual; pre_ff_norm(residual) -> hidden_states - # Also need raw (unfused) residual for router and pre_ff_norm_2 - hidden_states, residual = self.pre_feedforward_layernorm( - hidden_states, residual - ) # For MoE: router and pre_ff_norm_2 need the unfused residual # (which is now updated to post_attn_out + old_residual) moe_input = residual # Dense MLP branch - hidden_states_1 = self.mlp(hidden_states) + hidden_states_1 = self.ffn_boundary.sum_part( + self.mlp(hidden_states), forward_batch, SumGroup.TP + ) # MoE branch: router sees residual (= post_attn_out + old_residual) router_logits = self.router(moe_input) hidden_states_2 = self.pre_feedforward_layernorm_2(moe_input) - hidden_states_2 = self.moe(hidden_states_2, router_logits) + hidden_states_2 = self.ffn_boundary.sum_part( + self.moe(hidden_states_2, router_logits), + forward_batch, + SumGroup.MOE_OUTPUT, + ) # Fused: (rmsnorm(rmsnorm(h1,w1) + rmsnorm(h2,w2), w3) + residual) * scalar if ( @@ -722,7 +776,7 @@ class Gemma4DecoderLayer(nn.Module): norm2.variance_epsilon, norm3.variance_epsilon, ) - return hidden_states, None + return hidden_states hidden_states_1 = self.post_feedforward_layernorm_1(hidden_states_1) hidden_states_2 = self.post_feedforward_layernorm_2(hidden_states_2) @@ -730,11 +784,9 @@ class Gemma4DecoderLayer(nn.Module): # Combine branches hidden_states = hidden_states_1 + hidden_states_2 else: - # Fuse: hidden_states + residual -> residual; pre_ff_norm(residual) -> hidden_states - hidden_states, residual = self.pre_feedforward_layernorm( - hidden_states, residual + hidden_states = self.ffn_boundary.sum_part( + self.mlp(hidden_states), forward_batch, SumGroup.TP ) - hidden_states = self.mlp(hidden_states) if ( not self.has_ple @@ -766,7 +818,7 @@ class Gemma4DecoderLayer(nn.Module): hidden_states = hidden_states + per_layer_contribution hidden_states = hidden_states * self.layer_scalar - return hidden_states, None + return hidden_states class Gemma4TextModel(PreTrainedModel): @@ -977,49 +1029,45 @@ class Gemma4TextModel(PreTrainedModel): input_embeds, per_layer_inputs ) hidden_states = input_embeds + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None, ( "pp_proxy_tensors is required on non-first PP ranks" ) - hidden_states = pp_proxy_tensors["hidden_states"] + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( + pp_proxy_tensors, forward_batch + ) # PLE inputs were computed on rank 0 and forwarded along the # pipeline; non-PLE models simply omit the key. per_layer_inputs = pp_proxy_tensors.tensors.get("per_layer_inputs", None) - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() num_layers = self.config.num_hidden_layers for layer_idx in range(self.start_layer, self.end_layer): - if layer_idx in self.layers_to_capture: - aux_hidden_states.append(hidden_states) - if per_layer_inputs is not None: per_layer_input = per_layer_inputs[:, layer_idx, :] else: per_layer_input = None - layer = self.layers[layer_idx] - layer_outputs = layer( + hidden_states = self.layers[layer_idx]( positions=positions, hidden_states=hidden_states, per_layer_input=per_layer_input, forward_batch=forward_batch, + capture_output=aux_hidden_states.capture + if layer_idx in self.layers_to_capture + else None, **kwargs, ) - hidden_states = layer_outputs[0] - # Gemma4DecoderLayer.forward always returns (hidden_states, None); - # the residual is fused inside the layer, so nothing to thread. if not self.pp_group.is_last_rank: - # cuda_graph_runner allocates a fixed PP-proxy schema of - # {hidden_states, residual} and KeyErrors if a model omits a key. - # Gemma4 fuses the residual inside each layer so we don't have a - # standalone tensor to forward; emit a zero placeholder instead so - # graph replay can still copy it. The receiving stage never reads - # this key. - proxy = { - "hidden_states": hidden_states, - "residual": torch.zeros_like(hidden_states), - } + # Each layer writes the whole stream at its FFN exit, so it goes as + # hidden_states alone. cuda_graph_runner allocates a fixed PP-proxy + # schema of {hidden_states, residual} and KeyErrors if a model + # omits a key, so a zero placeholder fills the residual; the + # receiving stage never reads it. + proxy = residual_batch.to_pp(hidden_states, forward_batch).tensors + proxy["residual"] = torch.zeros_like(hidden_states) if per_layer_inputs is not None: proxy["per_layer_inputs"] = per_layer_inputs return PPProxyTensors(proxy) @@ -1030,7 +1078,9 @@ class Gemma4TextModel(PreTrainedModel): if num_layers in self.layers_to_capture: aux_hidden_states.append(hidden_states) - hidden_states = self.norm(hidden_states) + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, self.norm + ) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/iquest_q1.py b/python/sglang/srt/models/iquest_q1.py index ed613b2427..8bc444f531 100644 --- a/python/sglang/srt/models/iquest_q1.py +++ b/python/sglang/srt/models/iquest_q1.py @@ -9,6 +9,17 @@ from transformers import PretrainedConfig from triton.language.extra import libdevice from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.layer_boundary import ( + declare_attn, + declare_ffn, + make_stages, +) +from sglang.srt.layers.layer_boundary.output import OutputTransform +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual.add_norm import ( + UNFUSED_NORM_READOUT, + UnfusedNormReadout, +) from sglang.srt.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, @@ -180,8 +191,10 @@ class IQuestQ1MoEBlock(nn.Module): layer_id: int, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", + reduce_results: bool = True, ): super().__init__() + self.reduce_results = reduce_results self.hidden_size = config.hidden_size self.num_experts = config.num_experts self.top_k = config.num_experts_per_tok @@ -215,7 +228,8 @@ class IQuestQ1MoEBlock(nn.Module): router_logits, _ = self.gate(hidden_states.float()) topk_output = self.topk(hidden_states, router_logits) output = self.experts(hidden_states, topk_output) - output = post_experts_all_reduce(output) + if self.reduce_results: + output = post_experts_all_reduce(output) return output.reshape(original_shape) @@ -225,6 +239,7 @@ class IQuestQ1DenseMLP(nn.Module): config: PretrainedConfig, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", + reduce_results: bool = True, ): super().__init__() self.gate_up_proj = MergedColumnParallelLinear( @@ -239,6 +254,7 @@ class IQuestQ1DenseMLP(nn.Module): config.hidden_size, bias=False, quant_config=quant_config, + reduce_results=reduce_results, prefix=add_prefix("down_proj", prefix), ) if config.hidden_act != "silu": @@ -261,6 +277,7 @@ class IQuestQ1Attention(nn.Module): layer_id: int, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", + reduce_results: bool = True, ): super().__init__() parallel = get_parallel() @@ -303,6 +320,7 @@ class IQuestQ1Attention(nn.Module): quant_config=quant_config, tp_rank=attn_tp_rank, tp_size=attn_tp_size, + reduce_results=reduce_results, prefix=add_prefix("o_proj", prefix), ) self.q_norm = IQuestQ1RMSNorm(self.head_dim, eps=config.rms_norm_eps) @@ -397,6 +415,25 @@ class IQuestQ1Attention(nn.Module): return output +class _NormReplacesResidualRead(UnfusedNormReadout): + """After the update, the norm of the stream is both the stage's input and + the new stream: IQuest-Q1 layers after the first add their attention + output to the normalized input.""" + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + hidden_states, _ = super().read( + residual, norm, quant_format, post_residual_addition + ) + return hidden_states, hidden_states + + +_NORM_REPLACES_RESIDUAL = _NormReplacesResidualRead() + + +def _is_moe_layer(config: PretrainedConfig, layer_id: int) -> bool: + return layer_id not in (config.mlp_only_layers or []) + + class IQuestQ1DecoderLayer(nn.Module): def __init__( self, @@ -420,12 +457,14 @@ class IQuestQ1DecoderLayer(nn.Module): layer_id=layer_id, quant_config=quant_config, prefix=add_prefix("self_attn", prefix), + reduce_results=False, ) - if layer_id in (config.mlp_only_layers or []): + if not _is_moe_layer(config, layer_id): self.mlp = IQuestQ1DenseMLP( config=config, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) else: self.mlp = IQuestQ1MoEBlock( @@ -433,6 +472,7 @@ class IQuestQ1DecoderLayer(nn.Module): layer_id=layer_id, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) self.is_first_layer = layer_id == 0 if self.is_first_layer: @@ -444,6 +484,43 @@ class IQuestQ1DecoderLayer(nn.Module): else: self.attn_out_scale = config.attn_out_scale self.ffn_out_scale = config.ffn_out_scale + # Intentional: layer 0 adds the raw input and an FFN output norm, later layers + # add the normalized input without one; the MTP draft follows layer 0. + moe = _is_moe_layer(config, layer_id) + self.attn_boundary, self.ffn_boundary = make_stages( + ( + declare_attn( + read=UNFUSED_NORM_READOUT + if self.is_first_layer + else _NORM_REPLACES_RESIDUAL, + output_transform=OutputTransform(self._attn_output), + ), + self.attention_norm, + ), + ( + declare_ffn( + sparse=moe, + next_layer_sparse=_is_moe_layer(config, layer_id + 1), + read=UNFUSED_NORM_READOUT, + output_transform=OutputTransform(self._ffn_output), + ), + self.feed_forward_norm, + ), + previous=declare_ffn( + sparse=_is_moe_layer(config, layer_id - 1), next_layer_sparse=moe + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, + ) + + def _attn_output(self, attn_output: torch.Tensor) -> torch.Tensor: + return self.attn_out_norm(attn_output) * self.attn_out_scale + + def _ffn_output(self, mlp_output: torch.Tensor) -> torch.Tensor: + if self.is_first_layer: + return self.ffn_out_norm(mlp_output) * self.ffn_out_scale + return mlp_output * self.ffn_out_scale def forward( self, @@ -451,23 +528,13 @@ class IQuestQ1DecoderLayer(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, ) -> torch.Tensor: - # Intentional: layer 0 adds the raw input and an FFN output norm, later layers - # add the normalized input without one; the MTP draft follows layer 0. - if self.is_first_layer: - norm_hidden_states = self.attention_norm(hidden_states) - attn_output = self.self_attn(positions, norm_hidden_states, forward_batch) - hidden_states = hidden_states + self.attn_out_norm(attn_output) * ( - self.attn_out_scale - ) - mlp_output = self.mlp(self.feed_forward_norm(hidden_states)) - return hidden_states + self.ffn_out_norm(mlp_output) * self.ffn_out_scale - norm_hidden_states = self.attention_norm(hidden_states) - attn_output = self.self_attn(positions, norm_hidden_states, forward_batch) - hidden_states = norm_hidden_states + self.attn_out_norm(attn_output) * ( - self.attn_out_scale - ) - mlp_output = self.mlp(self.feed_forward_norm(hidden_states)) - return hidden_states + mlp_output * self.ffn_out_scale + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) + attn_output = self.self_attn(positions, hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(attn_output, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + mlp_output = self.mlp(hidden_states) + return ffn_exit.finish(mlp_output) class IQuestQ1Model(nn.Module): @@ -507,9 +574,11 @@ class IQuestQ1Model(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds + residual_batch.start(forward_batch) for layer in self.layers: hidden_states = layer(positions, hidden_states, forward_batch) - return self.norm(hidden_states) + hidden_states = residual_batch.fold(hidden_states, forward_batch) + return self.norm(residual_batch.take_output(hidden_states, forward_batch)) _FUSED_SHARD_LAYOUT = ( diff --git a/python/sglang/srt/models/zaya.py b/python/sglang/srt/models/zaya.py index a8fe43d8e8..f6f1111d69 100644 --- a/python/sglang/srt/models/zaya.py +++ b/python/sglang/srt/models/zaya.py @@ -52,9 +52,13 @@ import torch.nn.functional as F from torch import nn from sglang.srt.configs.zaya import ZayaConfig -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, +from sglang.srt.layers.layer_boundary import ( + declare_attn, + declare_ffn, + make_stages, ) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -64,6 +68,7 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.topk import StandardTopKOutput +from sglang.srt.layers.moe.utils import adds_replicated_output_to_partial from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope @@ -120,6 +125,72 @@ class ResidualScaling(nn.Module): return residual, hidden_states +class _ResidualMergeRead: + """How a ZAYA1 stage reads its input: its ResidualScaling, when the config + has one, on the incoming contribution and the fp32 residual, their fp32 + sum as the new residual, and the input norm of that sum in the model + dtype. The producer's update must be a plain add.""" + + is_plain_norm = False + reads_before_dp_gather = False + + def __init__(self, res_scale: Optional[ResidualScaling]): + self.res_scale = res_scale + + def _merge(self, residual, hidden_states): + if self.res_scale is not None: + residual, hidden_states = self.res_scale(residual, hidden_states) + if residual is None: + return hidden_states.float() + return residual.float() + hidden_states.float() + + def init_residual(self, hidden_states): + return self._merge(None, hidden_states) + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + if quant_format or post_residual_addition is not None: + raise NotImplementedError( + f"a ZAYA1 read with {quant_format=} or a post-residual addition" + ) + hidden_states = _apply_norm_with_fp32_residual( + norm, residual, norm.weight.dtype + ) + return hidden_states, residual + + def update_and_read( + self, + update, + hidden_states, + residual, + norm, + quant_format="", + post_residual_addition=None, + ): + if not update.is_plain_add: + raise NotImplementedError("a ZAYA1 read after a non-plain update") + return self.read( + self._merge(residual, hidden_states), + norm, + quant_format, + post_residual_addition, + ) + + +class _MergedNorm: + """The final norm, in the norm calling convention, read through the same + merge as the layers.""" + + def __init__(self, read: _ResidualMergeRead, norm: nn.Module): + self.read = read + self.norm = norm + + def __call__(self, hidden_states, residual=None): + if residual is None: + residual = self.read.init_residual(hidden_states) + return self.read.read(residual, self.norm)[0] + return self.read.update_and_read(PLAIN_ADD, hidden_states, residual, self.norm) + + def _apply_norm_with_fp32_residual( norm: nn.Module, residual: torch.Tensor, @@ -874,14 +945,14 @@ class ZayaAttention(nn.Module): prefix=add_prefix("qkv", prefix), ) - # RowParallel o_proj: per-rank input is the rank's q heads, full - # output is replicated via the end-of-forward all-reduce. + # RowParallel o_proj: per-rank input is the rank's q heads; the next + # stage's input completes the sum. self.o_proj = RowParallelLinear( self.q_dim_full, self.hidden_size, bias=bool(getattr(config, "attention_bias", False)), input_is_parallel=True, - reduce_results=True, + reduce_results=False, quant_config=quant_config, prefix=add_prefix("o_proj", prefix), ) @@ -1084,37 +1155,36 @@ def mod_premask_experts( indices: torch.Tensor, num_moe_experts: int, ) -> tuple[torch.Tensor, torch.Tensor]: - """Mask the (per-rank, pre-all-reduce) expert output for the MOD skip path. + """Mask the (per-rank, partial) expert output for the MOD skip path. Returns ``(mod_mask, masked_experts)`` where ``mod_mask`` is ``1`` for tokens routed to a real expert and ``0`` for tokens routed to the skip slot (``indices == num_moe_experts``), and ``masked_experts = mod_mask * experts_out``. - The masking is applied *before* the cross-rank all-reduce so the single - reduction yields ``mask · sum_r(partial_r) = mask · experts_out_full`` - without the replicated ``mod_out`` term being summed ``tp_size`` times. - Pairs with :func:`mod_blend`, which adds the skip-path term back after the - reduce. Kept as a free function so the MOD math is unit-testable without a - live ``torch.distributed`` group. + The mask is replicated, so the TP sum of the masked partials is + ``mask · experts_out_full``. Pairs with :func:`mod_blend`, which adds the + skip-path term. Kept as a free function so the MOD math is unit-testable + without a live ``torch.distributed`` group. """ mod_mask = (indices != num_moe_experts).to(experts_out.dtype) return mod_mask, mod_mask * experts_out def mod_blend( - masked_experts_reduced: torch.Tensor, + masked_experts: torch.Tensor, mod_mask: torch.Tensor, mod_out: torch.Tensor, ) -> torch.Tensor: - """Combine the already-all-reduced masked expert output with the skip path. + """Add the skip path, weighted by ``(1 - mod_mask)``, to the masked expert + output. ``mod_out`` (the skip-expert residual, ``hidden_states * prob``) is - replicated on every rank, so it is folded in here -- after the reduce of - ``masked_experts`` -- weighted by ``(1 - mod_mask)``. See - :func:`mod_premask_experts`. + replicated on every rank. While ``masked_experts`` is still a partial TP + sum, only one rank may add it (:func:`adds_replicated_output_to_partial`), + so that the sum counts it once. See :func:`mod_premask_experts`. """ - return masked_experts_reduced + (1.0 - mod_mask) * mod_out + return masked_experts + (1.0 - mod_mask) * mod_out class ZayaBlock(nn.Module): @@ -1201,25 +1271,21 @@ class ZayaBlock(nn.Module): experts_out = self.experts(hidden_states, topk_out) # ``mod_out`` is computed identically on every TP rank (both - # ``hidden_states`` and ``probs`` are replicated). Fold the skip - # mask into the per-rank partial experts output *before* - # all-reduce so the single reduction yields: + # ``hidden_states`` and ``probs`` are replicated). The output is a + # partial TP sum that the stage boundary completes, so only one + # rank adds the skip path: # sum_r(mask · partial_r) + (1 - mask) · mod_out # = mask · experts_out_full + (1 - mask) · mod_out - # without double-counting ``mod_out`` by tp_size. The two steps are - # ``mod_premask_experts`` / ``mod_blend`` so the math is testable - # without a live distributed group. + # The two steps are ``mod_premask_experts`` / ``mod_blend`` so the + # math is testable without a live distributed group. mod_out = hidden_states * probs - mod_mask, masked_experts = mod_premask_experts( + mod_mask, hidden_out = mod_premask_experts( experts_out, indices, self.num_moe_experts ) - if self.tp_size > 1: - masked_experts = tensor_model_parallel_all_reduce(masked_experts) - hidden_out = mod_blend(masked_experts, mod_mask, mod_out) + if adds_replicated_output_to_partial(): + hidden_out = mod_blend(hidden_out, mod_mask, mod_out) else: hidden_out = self.experts(hidden_states, topk_out) - if self.tp_size > 1: - hidden_out = tensor_model_parallel_all_reduce(hidden_out) return hidden_out, router_hs_next @@ -1254,6 +1320,14 @@ class ZayaDecoderATTLayer(nn.Module): self.res_scale = ResidualScaling(config, layer_id) else: self.res_scale = None + (self.attn_boundary,) = make_stages( + (declare_attn(read=_ResidualMergeRead(self.res_scale)), self.input_norm), + previous=declare_ffn(sparse=True, next_layer_sparse=True) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, + ) + self.entry_boundary = self.attn_boundary @staticmethod def _build_norm(config: ZayaConfig) -> nn.Module: @@ -1266,27 +1340,14 @@ class ZayaDecoderATTLayer(nn.Module): def forward( self, hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], positions: torch.Tensor, forward_batch: ForwardBatch, prev_router_hidden_states: Optional[torch.Tensor] = None, - ) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - target_dtype = ( - self.input_norm.weight.dtype - if isinstance(self.input_norm, RMSNorm) - else hidden_states.dtype - ) - if self.res_scale is not None: - residual, hidden_states = self.res_scale(residual, hidden_states) - if residual is not None: - residual = residual.float() + hidden_states.float() - else: - residual = hidden_states.float() - hidden_states = _apply_norm_with_fp32_residual( - self.input_norm, residual, target_dtype - ) + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) hidden_states = self.self_attn(hidden_states, positions, forward_batch) - return hidden_states, residual, prev_router_hidden_states + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + return hidden_states, prev_router_hidden_states class ZayaDecoderMLPLayer(nn.Module): @@ -1314,33 +1375,34 @@ class ZayaDecoderMLPLayer(nn.Module): self.res_scale = ResidualScaling(config, layer_id) else: self.res_scale = None + (self.ffn_boundary,) = make_stages( + ( + declare_ffn( + sparse=True, + next_layer_sparse=True, + read=_ResidualMergeRead(self.res_scale), + ), + self.input_norm, + ), + previous=declare_attn(), + terminal=layer_id == config.num_hidden_layers - 1, + ) + self.entry_boundary = self.ffn_boundary def forward( self, hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], positions: torch.Tensor, forward_batch: ForwardBatch, prev_router_hidden_states: Optional[torch.Tensor] = None, - ) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - target_dtype = ( - self.input_norm.weight.dtype - if isinstance(self.input_norm, RMSNorm) - else hidden_states.dtype - ) - if self.res_scale is not None: - residual, hidden_states = self.res_scale(residual, hidden_states) - if residual is not None: - residual = residual.float() + hidden_states.float() - else: - residual = hidden_states.float() - hidden_states = _apply_norm_with_fp32_residual( - self.input_norm, residual, target_dtype - ) - hidden_states, prev_router_hidden_states = self.zaya_block( - hidden_states, prev_router_hidden_states - ) - return hidden_states, residual, prev_router_hidden_states + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + hidden_states, prev_router_hidden_states = self.zaya_block( + hidden_states, prev_router_hidden_states + ) + hidden_states = ffn_exit.finish(hidden_states) + return hidden_states, prev_router_hidden_states # --------------------------------------------------------------------------- @@ -1412,6 +1474,9 @@ class ZayaModel(nn.Module): self.res_scale = ResidualScaling(config, config.num_hidden_layers) else: self.res_scale = None + self._final_merged_norm = _MergedNorm( + _ResidualMergeRead(self.res_scale), self.final_norm + ) else: self.final_norm = PPMissingLayer() self.res_scale = None @@ -1429,46 +1494,27 @@ class ZayaModel(nn.Module): hidden_states = inputs_embeds else: hidden_states = self.embed_tokens(input_ids) - residual = None + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states = self.layers[self.start_layer].entry_boundary.from_pp( + pp_proxy_tensors, forward_batch + ) prev_router_hidden_states: Optional[torch.Tensor] = None for i in range(self.start_layer, self.end_layer): - layer = self.layers[i] - hidden_states, residual, prev_router_hidden_states = layer( + hidden_states, prev_router_hidden_states = self.layers[i]( hidden_states=hidden_states, - residual=residual, positions=positions, forward_batch=forward_batch, prev_router_hidden_states=prev_router_hidden_states, ) if not self.pp_group.is_last_rank: - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) - - if self.res_scale is not None: - residual, hidden_states = self.res_scale(residual, hidden_states) - target_dtype = ( - self.final_norm.weight.dtype - if isinstance(self.final_norm, RMSNorm) - else hidden_states.dtype + return residual_batch.to_pp(hidden_states, forward_batch) + return residual_batch.final_norm( + hidden_states, forward_batch, self._final_merged_norm ) - if residual is not None: - merged = hidden_states.float() + residual.float() - else: - merged = hidden_states.float() - hidden_states = _apply_norm_with_fp32_residual( - self.final_norm, merged, target_dtype - ) - return hidden_states class ZayaForCausalLM(nn.Module): diff --git a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py index 931640eb5b..b8463e2a37 100644 --- a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py +++ b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py @@ -18,11 +18,13 @@ from sglang.srt.layers.layer_boundary import exit as exits from sglang.srt.layers.layer_boundary import ( make_stages, ) +from sglang.srt.layers.layer_boundary import stage as stages from sglang.srt.layers.layer_boundary.contracts import BatchVariant from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion from sglang.srt.layers.layer_boundary.layout import SumGroup from sglang.srt.layers.layer_boundary.prepare import _dispatch_by_update from sglang.srt.layers.layer_boundary.residual import batch +from sglang.srt.layers.layer_boundary.residual.add_norm import REPLACE_AT_EXIT from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.ci.ci_register import register_cpu_ci @@ -204,6 +206,49 @@ class TestBoundaryIntegrations(unittest.TestCase): with self.assertRaises(ValueError): declare_attn(reduction=ProducerReduction.TAIL_AFTER_SUM) + def test_ffn_writing_the_next_stream_hands_on_a_complete_output(self): + # It has each part's sum completed (sum_part) before building the + # stream from the parts, so its exit owes no sum and never defers one. + for attn_dp in (1, 2): + with ( + self.subTest(attn_dp=attn_dp), + fixture.planning(fixture.parallel_of(attn_dp=attn_dp, attn_tp=2)), + ): + for sparse in (False, True): + _, ffn = make_stages( + (declare_attn(), fixture.Norm()), + ( + declare_ffn(sparse=sparse, update=REPLACE_AT_EXIT), + fixture.Norm(), + ), + previous=declare_ffn(sparse=sparse, update=REPLACE_AT_EXIT), + ) + for variant, path in ffn.plan.paths.items(): + self.assertIsNone(path.output.group, (sparse, variant)) + self.assertFalse(path.output.may_defer_to_next) + + def test_only_an_ffn_writing_the_next_stream_sums_its_parts(self): + summed = Mock(side_effect=lambda value, *args, **kwargs: value * 2) + fb = SimpleNamespace() + with ( + fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=2)), + patch.object(stages, "sum_output", summed), + ): + for update in (PLAIN_ADD, REPLACE_AT_EXIT): + _, ffn = make_stages( + (declare_attn(), fixture.Norm()), + (declare_ffn(update=update), fixture.Norm()), + ) + if update is PLAIN_ADD: + with self.assertRaises(RuntimeError): + ffn.sum_part(torch.ones(2, 4), fb, SumGroup.TP) + continue + part = ffn.sum_part(torch.ones(2, 4), fb, SumGroup.MOE_OUTPUT) + torch.testing.assert_close(part, torch.full((2, 4), 2.0)) + summed.assert_called_once() + self.assertIs(summed.call_args.args[1], SumGroup.MOE_OUTPUT) + self.assertFalse(summed.call_args.kwargs["may_quantize"]) + def test_dense_decoder_forwards_capture_callback(self): from sglang.srt.models.llama4 import Llama4DecoderLayer from sglang.srt.models.qwen3 import Qwen3DecoderLayer diff --git a/test/registered/unit/models/test_zaya_mod_tp.py b/test/registered/unit/models/test_zaya_mod_tp.py index d5d45d6060..5589f2ccf8 100644 --- a/test/registered/unit/models/test_zaya_mod_tp.py +++ b/test/registered/unit/models/test_zaya_mod_tp.py @@ -1,11 +1,11 @@ """Numerical correctness test for ZAYA1 MoE + MOD under TP>1. -Background: the MOD (mixture-of-depths) skip-expert residual blend must be -combined with the experts output on the correct side of the cross-rank -all-reduce. ``mod_out = hidden_states * prob`` is replicated on every TP rank, -so all-reducing it would multiply it by ``tp_size``. The model therefore masks -the *per-rank partial* experts output before the reduce and only adds the -replicated ``mod_out`` afterwards: +Background: the MOD (mixture-of-depths) skip-expert residual blend must not +be counted once per TP rank. ``mod_out = hidden_states * prob`` is replicated +on every TP rank, while the experts output is a partial sum that the stage +boundary all-reduces. The model therefore masks each rank's partial experts +output and adds the replicated ``mod_out`` on one rank only, so the single +reduction yields: sum_r(mask · partial_r) + (1 - mask) · mod_out = mask · experts_out_full + (1 - mask) · mod_out @@ -50,17 +50,18 @@ def _real_tp_blend( partial_experts_per_rank: list[torch.Tensor], # one [T, H] per rank num_moe_experts: int, ) -> torch.Tensor: - """Production path: ``mod_premask_experts`` per rank -> simulated all-reduce - (sum) -> ``mod_blend``. Uses the exact helpers ``ZayaBlock.forward`` calls. + """Production path: ``mod_premask_experts`` per rank, ``mod_blend`` on rank + 0 only -> simulated all-reduce (sum). Uses the exact helpers + ``ZayaBlock.forward`` calls. """ mod_out = hidden_states * probs reduced = None - mod_mask = None - for partial in partial_experts_per_rank: + for rank, partial in enumerate(partial_experts_per_rank): mask, masked = mod_premask_experts(partial, indices, num_moe_experts) - mod_mask = mask + if rank == 0: + masked = mod_blend(masked, mask, mod_out) reduced = masked if reduced is None else reduced + masked - return mod_blend(reduced, mod_mask, mod_out) + return reduced def _buggy_old_tp_blend(