From 0bc074db83286449b96198eb340dacdf89041650 Mon Sep 17 00:00:00 2001 From: Xiaoyu Zhang <1182563586@qq.com> Date: Sun, 27 Sep 2026 16:57:12 +0800 Subject: [PATCH] [KDA+Kimi K3] Speed up SANA-Video residual gate add on H200 (#41305) Co-authored-by: kernel-design-agents <334018530+kernel-design-agents@users.noreply.github.com> --- .../kda_kernels/residual_gate_add_jit.py | 350 +++++++++++++++++- 1 file changed, 344 insertions(+), 6 deletions(-) diff --git a/python/sglang/kernels/kda_kernels/residual_gate_add_jit.py b/python/sglang/kernels/kda_kernels/residual_gate_add_jit.py index a3741b85c7..e5892561f8 100644 --- a/python/sglang/kernels/kda_kernels/residual_gate_add_jit.py +++ b/python/sglang/kernels/kda_kernels/residual_gate_add_jit.py @@ -4,6 +4,8 @@ import logging from typing import TYPE_CHECKING import torch +import triton +import triton.language as tl from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args from sglang.kernels.kda_kernels import _cuda_source @@ -55,12 +57,7 @@ def _fake_impl( ) -@register_custom_op( - op_name="diffusion_residual_gate_add", - mutates_args=[], - fake_impl=_fake_impl, -) -def _residual_gate_add_custom_op( +def _residual_gate_add_cuda_impl( residual: torch.Tensor, update: torch.Tensor, gate: torch.Tensor ) -> torch.Tensor: out = torch.empty_strided( @@ -85,6 +82,347 @@ def _residual_gate_add_custom_op( return out +@triton.jit +def _round16_f32(x, IS_BF16: tl.constexpr): + if IS_BF16: + bits = tl.inline_asm_elementwise( + "cvt.rn.bf16.f32 $0, $1;", "=h,r", [x], dtype=tl.int16, is_pure=True, pack=1 + ) + return bits.to(tl.bfloat16, bitcast=True).to(tl.float32) + else: + bits = tl.inline_asm_elementwise( + "cvt.rn.f16.f32 $0, $1;", "=h,r", [x], dtype=tl.int16, is_pure=True, pack=1 + ) + return bits.to(tl.float16, bitcast=True).to(tl.float32) + + +@triton.jit +def _store16(out_ptr, offs, value_f32, mask, IS_BF16: tl.constexpr): + if IS_BF16: + bits = tl.inline_asm_elementwise( + "cvt.rn.bf16.f32 $0, $1;", + "=h,r", + [value_f32], + dtype=tl.int16, + is_pure=True, + pack=1, + ) + tl.store(out_ptr + offs, bits.to(tl.bfloat16, bitcast=True), mask=mask) + else: + bits = tl.inline_asm_elementwise( + "cvt.rn.f16.f32 $0, $1;", + "=h,r", + [value_f32], + dtype=tl.int16, + is_pure=True, + pack=1, + ) + tl.store(out_ptr + offs, bits.to(tl.float16, bitcast=True), mask=mask) + + +@triton.jit +def _rga_flat( + out, + res, + upd, + gate, + numel, + hid, + MODE: tl.constexpr, + IS_BF16: tl.constexpr, + IS_16: tl.constexpr, + BLK: tl.constexpr, +): + pid = tl.program_id(0).to(tl.int64) + offs = pid * BLK + tl.arange(0, BLK).to(tl.int64) + mask = offs < numel + rv = tl.load(res + offs, mask=mask) + uv = tl.load(upd + offs, mask=mask) + if MODE == 1: + gv = tl.load(gate + (offs % hid), mask=mask) + elif MODE == 2: + gv = tl.load(gate + (offs // hid), mask=mask) + else: + gv = tl.load(gate + offs, mask=mask) + if IS_16: + p32 = uv.to(tl.float32) * gv.to(tl.float32) + pf = _round16_f32(p32, IS_BF16) + o32 = rv.to(tl.float32) + pf + _store16(out, offs, o32, mask, IS_BF16) + else: + p32 = uv * gv + o32 = rv + p32 + tl.store(out + offs, o32, mask=mask) + + +@triton.jit +def _rga_row( + out, + res, + upd, + gate, + hid: tl.constexpr, + IS_BF16: tl.constexpr, + EVG: tl.constexpr, + CG: tl.constexpr, + BLK: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + offs = tl.arange(0, BLK) + mask = offs < hid + if EVG: + gv = tl.load(gate + offs, mask=mask, eviction_policy="evict_last") + else: + gv = tl.load(gate + offs, mask=mask) + base = row * hid + if CG: + rv = tl.load(res + base + offs, mask=mask, cache_modifier=".cg") + uv = tl.load(upd + base + offs, mask=mask, cache_modifier=".cg") + else: + rv = tl.load(res + base + offs, mask=mask, eviction_policy="evict_first") + uv = tl.load(upd + base + offs, mask=mask, eviction_policy="evict_first") + p32 = uv.to(tl.float32) * gv.to(tl.float32) + pf = _round16_f32(p32, IS_BF16) + o32 = rv.to(tl.float32) + pf + _store16(out, base + offs, o32, mask, IS_BF16) + + +@triton.jit +def _rga_flat_m1( + out, + res, + upd, + gate, + numel, + hid: tl.constexpr, + IS_BF16: tl.constexpr, + BLK: tl.constexpr, +): + pid = tl.program_id(0).to(tl.int64) + offs = pid * BLK + tl.arange(0, BLK).to(tl.int64) + mask = offs < numel + rv = tl.load(res + offs, mask=mask, eviction_policy="evict_first") + uv = tl.load(upd + offs, mask=mask, eviction_policy="evict_first") + gv = tl.load(gate + (offs % hid), mask=mask, eviction_policy="evict_last") + p32 = uv.to(tl.float32) * gv.to(tl.float32) + pf = _round16_f32(p32, IS_BF16) + o32 = rv.to(tl.float32) + pf + _store16(out, offs, o32, mask, IS_BF16) + + +@triton.jit +def _rga_transposed( + out, + res, + upd, + gate, + tokens, + hid: tl.constexpr, + IS_BF16: tl.constexpr, + IS_16: tl.constexpr, + TILE: tl.constexpr, +): + # residual/out have logical [batch, tokens, hid] with physical strides (T*H, 1, T); + # update is contiguous. out[b,t,h] = res + upd * gate[h]. + # h-major tile indexing ([TILE_h, TILE_t]) makes the residual/output access + # coalesced along the physical stride-1 token dim; update (read once) takes + # the strided pattern instead. Measured faster than the t-major layout. + pid_t = tl.program_id(0).to(tl.int64) + pid_h = tl.program_id(1).to(tl.int64) + pid_b = tl.program_id(2).to(tl.int64) + t = pid_t * TILE + tl.arange(0, TILE).to(tl.int64) + h = pid_h * TILE + tl.arange(0, TILE).to(tl.int64) + mt = t < tokens + mh = h < hid + gv = tl.load(gate + h, mask=mh, other=0.0) + base = pid_b * tokens * hid + m2 = mh[:, None] & mt[None, :] + roffs = h[:, None] * tokens + t[None, :] + rv = tl.load(res + base + roffs, mask=m2, other=0.0) + uv = tl.load(upd + base + t[None, :] * hid + h[:, None], mask=m2, other=0.0) + if IS_16: + p32 = uv.to(tl.float32) * gv[:, None].to(tl.float32) + pf = _round16_f32(p32, IS_BF16) + o32 = rv.to(tl.float32) + pf + if IS_BF16: + bits = tl.inline_asm_elementwise( + "cvt.rn.bf16.f32 $0, $1;", + "=h,r", + [o32], + dtype=tl.int16, + is_pure=True, + pack=1, + ) + tl.store(out + base + roffs, bits.to(tl.bfloat16, bitcast=True), mask=m2) + else: + bits = tl.inline_asm_elementwise( + "cvt.rn.f16.f32 $0, $1;", + "=h,r", + [o32], + dtype=tl.int16, + is_pure=True, + pack=1, + ) + tl.store(out + base + roffs, bits.to(tl.float16, bitcast=True), mask=m2) + else: + tl.store(out + base + roffs, rv + uv * gv[:, None], mask=m2) + + +def _residual_gate_add_triton(residual, update, gate): + if not can_use_residual_gate_add_cuda(residual, update, gate): + raise RuntimeError("unsupported input for residual_gate_add CUDA") + out = torch.empty_strided( + residual.shape, residual.stride(), dtype=residual.dtype, device=residual.device + ) + is_bf16 = residual.dtype == torch.bfloat16 + dtype16 = residual.dtype in (torch.float16, torch.bfloat16) + + if _is_transposed_dense_residual(residual, update, gate): + tokens = residual.shape[1] + hid = residual.shape[2] + batch = residual.shape[0] + if tokens * hid * batch <= 65536: + tile, warps = 16, 4 + elif tokens * hid * batch >= 1 << 20: + tile, warps = 64, 8 + else: + tile, warps = 32, 8 + grid = (triton.cdiv(tokens, tile), triton.cdiv(hid, tile), batch) + _rga_transposed[grid]( + out, + residual, + update, + gate, + tokens, + hid, + is_bf16, + dtype16, + TILE=tile, + num_warps=warps, + ) + return out + + if not dtype16: + # fp32: exact fp32 math, plain Triton elementwise + numel = residual.numel() + hid = residual.shape[-1] + if gate.shape == residual.shape: + mode = 0 + elif _is_row_broadcast_gate(residual, gate): + mode = 1 + else: + mode = 2 + if numel <= 8192: + _rga_flat[(triton.cdiv(numel, 256),)]( + out, + residual, + update, + gate, + numel, + hid, + mode, + False, + False, + BLK=256, + num_warps=4, + ) + else: + _rga_flat[(triton.cdiv(numel, 1024),)]( + out, + residual, + update, + gate, + numel, + hid, + mode, + False, + False, + BLK=1024, + num_warps=4, + ) + return out + + hid = residual.shape[-1] + numel = residual.numel() + if _is_row_broadcast_gate(residual, gate): + rows = numel // hid + if hid >= 4096 or (rows <= 512 and numel >= 1 << 20): + _rga_flat_m1[(triton.cdiv(numel, 1024),)]( + out, residual, update, gate, numel, hid, is_bf16, BLK=1024, num_warps=4 + ) + elif hid <= 16384: + blk = triton.next_power_of_2(hid) + _rga_row[(rows,)]( + out, + residual, + update, + gate, + hid, + is_bf16, + hid < 4608, + hid >= 4096, + BLK=blk, + num_warps=4 if hid >= 4608 else 8, + ) + else: + _rga_flat[(triton.cdiv(numel, 1024),)]( + out, + residual, + update, + gate, + numel, + hid, + 1, + is_bf16, + True, + BLK=1024, + num_warps=4, + ) + return out + mode = 0 if gate.shape == residual.shape else 2 + if numel <= 8192: + _rga_flat[(triton.cdiv(numel, 256),)]( + out, + residual, + update, + gate, + numel, + hid, + mode, + is_bf16, + True, + BLK=256, + num_warps=4, + ) + else: + _rga_flat[(triton.cdiv(numel, 1024),)]( + out, + residual, + update, + gate, + numel, + hid, + mode, + is_bf16, + True, + BLK=1024, + num_warps=4, + ) + return out + + +@register_custom_op( + op_name="diffusion_residual_gate_add", + mutates_args=[], + fake_impl=_fake_impl, +) +def _residual_gate_add_custom_op( + residual: torch.Tensor, update: torch.Tensor, gate: torch.Tensor +) -> torch.Tensor: + with torch.cuda.device(residual.device): + return _residual_gate_add_triton(residual, update, gate) + + def _gate_mode(residual: torch.Tensor, gate: torch.Tensor) -> int: """0 = full, 1 = broadcast row (hidden_size), 2 = per-token (rows).""" if gate.shape == residual.shape: