From 1e77db3215f132efb436679511067671f974a66f Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Thu, 8 Oct 2026 17:11:58 -0700 Subject: [PATCH] [Perf] Avoid gate copies in gated RMSNorm (#43227) --- .../ops/attention/fla/fused_norm_gate.py | 51 ++++++- .../attention/test_fused_norm_gate_strided.py | 139 ++++++++++++++++++ 2 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 test/registered/kernels/ops/attention/test_fused_norm_gate_strided.py diff --git a/python/sglang/kernels/ops/attention/fla/fused_norm_gate.py b/python/sglang/kernels/ops/attention/fla/fused_norm_gate.py index d13be4b877..60fb8f3f04 100644 --- a/python/sglang/kernels/ops/attention/fla/fused_norm_gate.py +++ b/python/sglang/kernels/ops/attention/fla/fused_norm_gate.py @@ -46,6 +46,10 @@ def layer_norm_gated_fwd_kernel( HAS_WEIGHT: tl.constexpr, HAS_BIAS: tl.constexpr, USE_GDC: tl.constexpr = False, + G_STRIDED: tl.constexpr = False, + G_HEADS: tl.constexpr = 1, + stride_g_tok=0, + stride_g_head=0, ): # PDL: x is the producer's output (e.g. the fused KDA verify kernel, which # triggers its dependents right after the o store), so every load sits @@ -98,8 +102,21 @@ def layer_norm_gated_fwd_kernel( b_y = b_y + b_b[None, :] # swish/sigmoid output gate - p_g = tl.make_block_ptr(g, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0)) - b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32) + if G_STRIDED: + # strided [tokens, G_HEADS, D] gate: row = token * G_HEADS + head + o_r = i_t * BT + tl.arange(0, BT) + m_r = o_r < T + g_off = (o_r // G_HEADS).to(tl.int64) * stride_g_tok + ( + o_r % G_HEADS + ) * stride_g_head + b_g = tl.load( + g + g_off[:, None] + o_d[None, :], + mask=m_r[:, None] & m_d[None, :], + other=0.0, + ).to(tl.float32) + else: + p_g = tl.make_block_ptr(g, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0)) + b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32) if ACTIVATION == "swish" or ACTIVATION == "silu": b_y = b_y * b_g * tl.sigmoid(b_g) elif ACTIVATION == "sigmoid": @@ -191,6 +208,7 @@ def layer_norm_gated_fwd( out_dtype: torch.dtype = None, residual_dtype: torch.dtype = None, is_rms_norm: bool = False, + g_strided: tuple[int, int, int] | None = None, # (heads, token stride, head stride) ): if residual is not None: residual_dtype = residual.dtype @@ -256,8 +274,20 @@ def layer_norm_gated_fwd( HAS_BIAS=bias is not None, num_warps=4, **pdl_kwargs, + **( + dict( + G_STRIDED=True, + G_HEADS=g_strided[0], + stride_g_tok=g_strided[1], + stride_g_head=g_strided[2], + ) + if g_strided is not None + else {} + ), ) else: + if g_strided is not None: + g = g.reshape(-1, D) layer_norm_gated_fwd_kernel1[(T,)]( x=x, g=g, @@ -302,7 +332,21 @@ class LayerNormGatedFunction(torch.autograd.Function): g_shape_og = g.shape # reshape input data into 2D tensor x = x.reshape(-1, x.shape[-1]) - g = g.reshape(-1, g.shape[-1]) + # A column slice of a fused projection can't be viewed as [rows, D]; + # pass its strides instead of copying. + g_strided = None + if ( + not g.is_contiguous() + and g.dim() >= 3 + and g.shape[-1] == x.shape[-1] + and g.shape[-1] <= 512 + and g.stride(-1) == 1 + and g.stride(-2) == g.shape[-1] + and all(n == 1 for n in g.shape[:-3]) + ): + g_strided = (g.shape[-2], g.stride(-3), g.stride(-2)) + else: + g = g.reshape(-1, g.shape[-1]).contiguous() if residual is not None: assert residual.shape == x_shape_og residual = residual.reshape(-1, residual.shape[-1]) @@ -321,6 +365,7 @@ class LayerNormGatedFunction(torch.autograd.Function): residual=residual, residual_dtype=residual_dtype, is_rms_norm=is_rms_norm, + g_strided=g_strided, ) ctx.save_for_backward(residual_out, g, weight, bias, mean, rstd) ctx.x_shape_og = x_shape_og diff --git a/test/registered/kernels/ops/attention/test_fused_norm_gate_strided.py b/test/registered/kernels/ops/attention/test_fused_norm_gate_strided.py new file mode 100644 index 0000000000..9c163b8667 --- /dev/null +++ b/test/registered/kernels/ops/attention/test_fused_norm_gate_strided.py @@ -0,0 +1,139 @@ +"""A strided output gate (a column slice of a fused projection, as in Kimi-K3 +KDA) must give the same gated RMSNorm as a contiguous one, without a copy.""" + +import pytest +import torch + +from sglang.kernels.ops.attention.fla.fused_norm_gate import FusedRMSNormGated +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +_HEADS, _HEAD_DIM, _PROJ_WIDTH = 12, 128, 6144 + + +def _strided_gate(num_tokens): + proj = torch.randn(num_tokens, _PROJ_WIDTH, device="cuda", dtype=torch.bfloat16) + return proj[:, _PROJ_WIDTH - _HEADS * _HEAD_DIM :].unflatten(-1, (-1, _HEAD_DIM)) + + +@pytest.mark.parametrize("activation", ["sigmoid", "swish"]) +@pytest.mark.parametrize("num_tokens", [1, 2, 8, 33, 1000]) +@torch.inference_mode() +def test_strided_gate_matches_contiguous(activation, num_tokens): + torch.manual_seed(num_tokens) + norm = FusedRMSNormGated( + _HEAD_DIM, eps=1e-6, activation=activation, device="cuda", dtype=torch.bfloat16 + ) + torch.nn.init.normal_(norm.weight, 1.0, 0.1) + gate = _strided_gate(num_tokens) + x = torch.randn( + 1, num_tokens, _HEADS, _HEAD_DIM, device="cuda", dtype=torch.bfloat16 + ) + torch.testing.assert_close( + norm(x.clone(), gate), norm(x.clone(), gate.contiguous()), rtol=0, atol=0 + ) + + +@pytest.mark.parametrize("head_dim", [64, 128, 256, 512]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) +@pytest.mark.parametrize("activation", ["sigmoid", "silu"]) +@torch.inference_mode() +def test_strided_gate_widths_and_dtypes(head_dim, dtype, activation): + torch.manual_seed(0) + tokens, heads = 11, 3 + proj = torch.randn(tokens, 4 * heads * head_dim, device="cuda", dtype=dtype) + gate = proj[:, 3 * heads * head_dim :].unflatten(-1, (heads, head_dim)) + x = torch.randn(tokens, heads, head_dim, device="cuda", dtype=dtype) + norm = FusedRMSNormGated( + head_dim, activation=activation, device="cuda", dtype=dtype + ) + torch.nn.init.normal_(norm.weight, 1.0, 0.1) + torch.testing.assert_close( + norm(x.clone(), gate), norm(x.clone(), gate.contiguous()), rtol=0, atol=0 + ) + + +@pytest.mark.parametrize( + "layout", ["2d", "inner_stride", "head_stride", "batch", "wide"] +) +@torch.inference_mode() +def test_unsupported_gate_layouts_fall_back(layout): + torch.manual_seed(0) + dim = 1024 if layout == "wide" else 128 + shape = (2, 3, 4, dim) if layout == "batch" else (3, 4, dim) + if layout == "inner_stride": + gate = torch.randn(*shape[:-1], 2 * dim, device="cuda", dtype=torch.bfloat16)[ + ..., ::2 + ] + elif layout == "head_stride": + gate = torch.randn(3, 8, dim, device="cuda", dtype=torch.bfloat16)[:, ::2, :] + else: + proj = torch.randn( + *shape[:-2], 4 * shape[-2] * dim, device="cuda", dtype=torch.bfloat16 + ) + gate = proj[..., -shape[-2] * dim :].unflatten(-1, (shape[-2], dim)) + if layout == "2d": + gate = gate[:, 0, :] + x = torch.randn_like(gate) + norm = FusedRMSNormGated( + dim, activation="sigmoid", device="cuda", dtype=torch.bfloat16 + ) + torch.nn.init.ones_(norm.weight) + torch.testing.assert_close( + norm(x.clone(), gate), norm(x.clone(), gate.contiguous()), rtol=0, atol=0 + ) + + +@pytest.mark.parametrize("with_residual", [False, True]) +@torch.inference_mode() +def test_strided_gate_cuda_graph_and_prenorm(with_residual): + norm = FusedRMSNormGated( + _HEAD_DIM, activation="sigmoid", device="cuda", dtype=torch.bfloat16 + ) + torch.nn.init.ones_(norm.weight) + gate = _strided_gate(8) + x = torch.randn(1, 8, _HEADS, _HEAD_DIM, device="cuda", dtype=torch.bfloat16) + residual = torch.randn_like(x) if with_residual else None + expected = norm(x.clone(), gate.contiguous(), residual=residual, prenorm=True) + static_x = x.clone() + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + norm(static_x, gate, residual=residual, prenorm=True) + torch.cuda.current_stream().wait_stream(stream) + static_x.copy_(x) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual = norm(static_x, gate, residual=residual, prenorm=True) + static_x.copy_(x) + graph.replay() + torch.cuda.synchronize() + for result, reference in zip(actual, expected): + torch.testing.assert_close(result, reference, rtol=0, atol=0) + + +@torch.inference_mode() +def test_strided_gate_launches_no_copy(): + norm = FusedRMSNormGated( + _HEAD_DIM, eps=1e-6, activation="sigmoid", device="cuda", dtype=torch.bfloat16 + ) + gate = _strided_gate(8) + x = torch.randn(1, 8, _HEADS, _HEAD_DIM, device="cuda", dtype=torch.bfloat16) + norm(x, gate) + torch.cuda.synchronize() + with torch.profiler.profile( + activities=[torch.profiler.ProfilerActivity.CUDA] + ) as prof: + norm(x, gate) + torch.cuda.synchronize() + kernels = [ + e.name for e in prof.events() if e.device_type == torch.autograd.DeviceType.CUDA + ] + assert kernels and not any("copy" in k for k in kernels), kernels + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__]))