[Fix] GigaChat 3.5: apply the sandwich norms to the complete sums (#42310)
This commit is contained in:
@@ -2721,9 +2721,11 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
post_attention_layernorm: nn.Module,
|
||||
qkv_latent_func: Optional[Callable],
|
||||
output=None,
|
||||
attn_output=None,
|
||||
):
|
||||
"""The stage boundaries for this layer's norms; they choose their
|
||||
steps from them at construction."""
|
||||
steps from them at construction. ``attn_output`` and ``output``: the
|
||||
attention's and the FFN's output transforms, if any."""
|
||||
fusions = None
|
||||
if (
|
||||
not get_parallel().enable_prefill_cp
|
||||
@@ -2736,7 +2738,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
fusions = CuteDSLFusion()
|
||||
attn_boundary, ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
declare_attn(output_transform=attn_output),
|
||||
input_layernorm,
|
||||
{"qkv_latent_func": qkv_latent_func, "fusions": fusions},
|
||||
),
|
||||
|
||||
@@ -130,7 +130,8 @@ def build_norm(
|
||||
|
||||
|
||||
class GigaChat35PassthroughNorm(nn.Module):
|
||||
"""Identity norm with the fused residual-add convention.
|
||||
"""Identity norm with the norm calling convention: ``forward(x)`` returns
|
||||
``x``, ``forward(x, residual)`` returns ``(residual + x, residual + x)``.
|
||||
|
||||
Used as the ``input_layernorm`` slot when a layer has no pre-norm
|
||||
(``layernorm_type="post"``); it only folds the residual so the
|
||||
@@ -146,29 +147,21 @@ class GigaChat35PassthroughNorm(nn.Module):
|
||||
if post_residual_addition is not None:
|
||||
x = x + post_residual_addition
|
||||
if residual is None:
|
||||
return x, x
|
||||
return x
|
||||
merged = x + residual
|
||||
return merged, merged
|
||||
|
||||
|
||||
class GigaChat35MlpPrepNorm(nn.Module):
|
||||
"""Pre-MLP norm wrapper that folds the post-attention (sandwich) norm.
|
||||
|
||||
Serves as the FFN boundary's norm so the
|
||||
optional ``post_self_attn_layernorm`` (applied to the attention output,
|
||||
before the residual add) and the pre-MLP ``post_attention_layernorm`` are
|
||||
both run during FFN input preparation, matching the per-layer math of the
|
||||
``pre_post`` sandwich exactly.
|
||||
"""Pre-MLP norm wrapper: adds the residual, then runs the optional pre-MLP
|
||||
``post_attention_layernorm`` on the sum, unfused. The attention's own
|
||||
``post_self_attn_layernorm`` is its declared output transform, which the
|
||||
FFN input runs before this.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pre_layernorm: Optional[nn.Module],
|
||||
post_layernorm: Optional[nn.Module],
|
||||
) -> None:
|
||||
def __init__(self, pre_layernorm: Optional[nn.Module]) -> None:
|
||||
super().__init__()
|
||||
self.pre_layernorm = pre_layernorm
|
||||
self.post_layernorm = post_layernorm
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -176,14 +169,11 @@ class GigaChat35MlpPrepNorm(nn.Module):
|
||||
residual: Optional[torch.Tensor] = None,
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
):
|
||||
if self.post_layernorm is not None:
|
||||
x = self.post_layernorm(x)
|
||||
if post_residual_addition is not None:
|
||||
x = x + post_residual_addition
|
||||
if residual is None:
|
||||
merged = x
|
||||
else:
|
||||
merged = x + residual
|
||||
return x if self.pre_layernorm is None else self.pre_layernorm(x)
|
||||
merged = x + residual
|
||||
if self.pre_layernorm is None:
|
||||
return merged, merged
|
||||
return self.pre_layernorm(merged), merged
|
||||
@@ -438,17 +428,19 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
|
||||
)
|
||||
mlp_prepare_layernorm = GigaChat35MlpPrepNorm(
|
||||
pre_layernorm=self.post_attention_layernorm if self._use_pre else None,
|
||||
post_layernorm=self.post_self_attn_layernorm,
|
||||
)
|
||||
# The stage boundaries choose their fused steps from their norms when built.
|
||||
self.attn_boundary, self.ffn_boundary = self._build_stages(
|
||||
input_layernorm=attn_prepare_layernorm,
|
||||
post_attention_layernorm=mlp_prepare_layernorm,
|
||||
qkv_latent_func=qkv_latent_func,
|
||||
attn_output=(
|
||||
OutputTransform(self.post_self_attn_layernorm)
|
||||
if self.post_self_attn_layernorm is not None
|
||||
else None
|
||||
),
|
||||
output=(
|
||||
OutputTransform(
|
||||
self.post_feedforward_layernorm, before_reduce_scatter=True
|
||||
)
|
||||
OutputTransform(self.post_feedforward_layernorm)
|
||||
if self.post_feedforward_layernorm is not None
|
||||
else None
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user