[Perf] Avoid gate copies in gated RMSNorm (#43227)

This commit is contained in:
Khoa Pham
2026-10-08 17:11:58 -07:00
committed by GitHub
parent 20d1f3f56a
commit 1e77db3215
2 changed files with 187 additions and 3 deletions
@@ -46,6 +46,10 @@ def layer_norm_gated_fwd_kernel(
HAS_WEIGHT: tl.constexpr,
HAS_BIAS: tl.constexpr,
USE_GDC: tl.constexpr = False,
G_STRIDED: tl.constexpr = False,
G_HEADS: tl.constexpr = 1,
stride_g_tok=0,
stride_g_head=0,
):
# PDL: x is the producer's output (e.g. the fused KDA verify kernel, which
# triggers its dependents right after the o store), so every load sits
@@ -98,8 +102,21 @@ def layer_norm_gated_fwd_kernel(
b_y = b_y + b_b[None, :]
# swish/sigmoid output gate
p_g = tl.make_block_ptr(g, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
if G_STRIDED:
# strided [tokens, G_HEADS, D] gate: row = token * G_HEADS + head
o_r = i_t * BT + tl.arange(0, BT)
m_r = o_r < T
g_off = (o_r // G_HEADS).to(tl.int64) * stride_g_tok + (
o_r % G_HEADS
) * stride_g_head
b_g = tl.load(
g + g_off[:, None] + o_d[None, :],
mask=m_r[:, None] & m_d[None, :],
other=0.0,
).to(tl.float32)
else:
p_g = tl.make_block_ptr(g, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
if ACTIVATION == "swish" or ACTIVATION == "silu":
b_y = b_y * b_g * tl.sigmoid(b_g)
elif ACTIVATION == "sigmoid":
@@ -191,6 +208,7 @@ def layer_norm_gated_fwd(
out_dtype: torch.dtype = None,
residual_dtype: torch.dtype = None,
is_rms_norm: bool = False,
g_strided: tuple[int, int, int] | None = None, # (heads, token stride, head stride)
):
if residual is not None:
residual_dtype = residual.dtype
@@ -256,8 +274,20 @@ def layer_norm_gated_fwd(
HAS_BIAS=bias is not None,
num_warps=4,
**pdl_kwargs,
**(
dict(
G_STRIDED=True,
G_HEADS=g_strided[0],
stride_g_tok=g_strided[1],
stride_g_head=g_strided[2],
)
if g_strided is not None
else {}
),
)
else:
if g_strided is not None:
g = g.reshape(-1, D)
layer_norm_gated_fwd_kernel1[(T,)](
x=x,
g=g,
@@ -302,7 +332,21 @@ class LayerNormGatedFunction(torch.autograd.Function):
g_shape_og = g.shape
# reshape input data into 2D tensor
x = x.reshape(-1, x.shape[-1])
g = g.reshape(-1, g.shape[-1])
# A column slice of a fused projection can't be viewed as [rows, D];
# pass its strides instead of copying.
g_strided = None
if (
not g.is_contiguous()
and g.dim() >= 3
and g.shape[-1] == x.shape[-1]
and g.shape[-1] <= 512
and g.stride(-1) == 1
and g.stride(-2) == g.shape[-1]
and all(n == 1 for n in g.shape[:-3])
):
g_strided = (g.shape[-2], g.stride(-3), g.stride(-2))
else:
g = g.reshape(-1, g.shape[-1]).contiguous()
if residual is not None:
assert residual.shape == x_shape_og
residual = residual.reshape(-1, residual.shape[-1])
@@ -321,6 +365,7 @@ class LayerNormGatedFunction(torch.autograd.Function):
residual=residual,
residual_dtype=residual_dtype,
is_rms_norm=is_rms_norm,
g_strided=g_strided,
)
ctx.save_for_backward(residual_out, g, weight, bias, mean, rstd)
ctx.x_shape_og = x_shape_og