diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 0145b36e14..4264b1934f 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -2721,9 +2721,11 @@ class DeepseekV2DecoderLayer(nn.Module): post_attention_layernorm: nn.Module, qkv_latent_func: Optional[Callable], output=None, + attn_output=None, ): """The stage boundaries for this layer's norms; they choose their - steps from them at construction.""" + steps from them at construction. ``attn_output`` and ``output``: the + attention's and the FFN's output transforms, if any.""" fusions = None if ( not get_parallel().enable_prefill_cp @@ -2736,7 +2738,7 @@ class DeepseekV2DecoderLayer(nn.Module): fusions = CuteDSLFusion() attn_boundary, ffn_boundary = make_stages( ( - declare_attn(), + declare_attn(output_transform=attn_output), input_layernorm, {"qkv_latent_func": qkv_latent_func, "fusions": fusions}, ), diff --git a/python/sglang/srt/models/gigachat35.py b/python/sglang/srt/models/gigachat35.py index 8da089590e..7a0ff1c91d 100644 --- a/python/sglang/srt/models/gigachat35.py +++ b/python/sglang/srt/models/gigachat35.py @@ -130,7 +130,8 @@ def build_norm( class GigaChat35PassthroughNorm(nn.Module): - """Identity norm with the fused residual-add convention. + """Identity norm with the norm calling convention: ``forward(x)`` returns + ``x``, ``forward(x, residual)`` returns ``(residual + x, residual + x)``. Used as the ``input_layernorm`` slot when a layer has no pre-norm (``layernorm_type="post"``); it only folds the residual so the @@ -146,29 +147,21 @@ class GigaChat35PassthroughNorm(nn.Module): if post_residual_addition is not None: x = x + post_residual_addition if residual is None: - return x, x + return x merged = x + residual return merged, merged class GigaChat35MlpPrepNorm(nn.Module): - """Pre-MLP norm wrapper that folds the post-attention (sandwich) norm. - - Serves as the FFN boundary's norm so the - optional ``post_self_attn_layernorm`` (applied to the attention output, - before the residual add) and the pre-MLP ``post_attention_layernorm`` are - both run during FFN input preparation, matching the per-layer math of the - ``pre_post`` sandwich exactly. + """Pre-MLP norm wrapper: adds the residual, then runs the optional pre-MLP + ``post_attention_layernorm`` on the sum, unfused. The attention's own + ``post_self_attn_layernorm`` is its declared output transform, which the + FFN input runs before this. """ - def __init__( - self, - pre_layernorm: Optional[nn.Module], - post_layernorm: Optional[nn.Module], - ) -> None: + def __init__(self, pre_layernorm: Optional[nn.Module]) -> None: super().__init__() self.pre_layernorm = pre_layernorm - self.post_layernorm = post_layernorm def forward( self, @@ -176,14 +169,11 @@ class GigaChat35MlpPrepNorm(nn.Module): residual: Optional[torch.Tensor] = None, post_residual_addition: Optional[torch.Tensor] = None, ): - if self.post_layernorm is not None: - x = self.post_layernorm(x) if post_residual_addition is not None: x = x + post_residual_addition if residual is None: - merged = x - else: - merged = x + residual + return x if self.pre_layernorm is None else self.pre_layernorm(x) + merged = x + residual if self.pre_layernorm is None: return merged, merged return self.pre_layernorm(merged), merged @@ -438,17 +428,19 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer): ) mlp_prepare_layernorm = GigaChat35MlpPrepNorm( pre_layernorm=self.post_attention_layernorm if self._use_pre else None, - post_layernorm=self.post_self_attn_layernorm, ) # The stage boundaries choose their fused steps from their norms when built. self.attn_boundary, self.ffn_boundary = self._build_stages( input_layernorm=attn_prepare_layernorm, post_attention_layernorm=mlp_prepare_layernorm, qkv_latent_func=qkv_latent_func, + attn_output=( + OutputTransform(self.post_self_attn_layernorm) + if self.post_self_attn_layernorm is not None + else None + ), output=( - OutputTransform( - self.post_feedforward_layernorm, before_reduce_scatter=True - ) + OutputTransform(self.post_feedforward_layernorm) if self.post_feedforward_layernorm is not None else None ),