[KDA+Kimi K3] Speed up SANA-Video residual gate add on H200 (#41305)

Co-authored-by: kernel-design-agents <334018530+kernel-design-agents@users.noreply.github.com>
This commit is contained in:
Xiaoyu Zhang
2026-09-27 16:57:12 +08:00
committed by GitHub
co-authored by kernel-design-agents
parent aa7a976807
commit 0bc074db83
@@ -4,6 +4,8 @@ import logging
from typing import TYPE_CHECKING
import torch
import triton
import triton.language as tl
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
from sglang.kernels.kda_kernels import _cuda_source
@@ -55,12 +57,7 @@ def _fake_impl(
)
@register_custom_op(
op_name="diffusion_residual_gate_add",
mutates_args=[],
fake_impl=_fake_impl,
)
def _residual_gate_add_custom_op(
def _residual_gate_add_cuda_impl(
residual: torch.Tensor, update: torch.Tensor, gate: torch.Tensor
) -> torch.Tensor:
out = torch.empty_strided(
@@ -85,6 +82,347 @@ def _residual_gate_add_custom_op(
return out
@triton.jit
def _round16_f32(x, IS_BF16: tl.constexpr):
if IS_BF16:
bits = tl.inline_asm_elementwise(
"cvt.rn.bf16.f32 $0, $1;", "=h,r", [x], dtype=tl.int16, is_pure=True, pack=1
)
return bits.to(tl.bfloat16, bitcast=True).to(tl.float32)
else:
bits = tl.inline_asm_elementwise(
"cvt.rn.f16.f32 $0, $1;", "=h,r", [x], dtype=tl.int16, is_pure=True, pack=1
)
return bits.to(tl.float16, bitcast=True).to(tl.float32)
@triton.jit
def _store16(out_ptr, offs, value_f32, mask, IS_BF16: tl.constexpr):
if IS_BF16:
bits = tl.inline_asm_elementwise(
"cvt.rn.bf16.f32 $0, $1;",
"=h,r",
[value_f32],
dtype=tl.int16,
is_pure=True,
pack=1,
)
tl.store(out_ptr + offs, bits.to(tl.bfloat16, bitcast=True), mask=mask)
else:
bits = tl.inline_asm_elementwise(
"cvt.rn.f16.f32 $0, $1;",
"=h,r",
[value_f32],
dtype=tl.int16,
is_pure=True,
pack=1,
)
tl.store(out_ptr + offs, bits.to(tl.float16, bitcast=True), mask=mask)
@triton.jit
def _rga_flat(
out,
res,
upd,
gate,
numel,
hid,
MODE: tl.constexpr,
IS_BF16: tl.constexpr,
IS_16: tl.constexpr,
BLK: tl.constexpr,
):
pid = tl.program_id(0).to(tl.int64)
offs = pid * BLK + tl.arange(0, BLK).to(tl.int64)
mask = offs < numel
rv = tl.load(res + offs, mask=mask)
uv = tl.load(upd + offs, mask=mask)
if MODE == 1:
gv = tl.load(gate + (offs % hid), mask=mask)
elif MODE == 2:
gv = tl.load(gate + (offs // hid), mask=mask)
else:
gv = tl.load(gate + offs, mask=mask)
if IS_16:
p32 = uv.to(tl.float32) * gv.to(tl.float32)
pf = _round16_f32(p32, IS_BF16)
o32 = rv.to(tl.float32) + pf
_store16(out, offs, o32, mask, IS_BF16)
else:
p32 = uv * gv
o32 = rv + p32
tl.store(out + offs, o32, mask=mask)
@triton.jit
def _rga_row(
out,
res,
upd,
gate,
hid: tl.constexpr,
IS_BF16: tl.constexpr,
EVG: tl.constexpr,
CG: tl.constexpr,
BLK: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
offs = tl.arange(0, BLK)
mask = offs < hid
if EVG:
gv = tl.load(gate + offs, mask=mask, eviction_policy="evict_last")
else:
gv = tl.load(gate + offs, mask=mask)
base = row * hid
if CG:
rv = tl.load(res + base + offs, mask=mask, cache_modifier=".cg")
uv = tl.load(upd + base + offs, mask=mask, cache_modifier=".cg")
else:
rv = tl.load(res + base + offs, mask=mask, eviction_policy="evict_first")
uv = tl.load(upd + base + offs, mask=mask, eviction_policy="evict_first")
p32 = uv.to(tl.float32) * gv.to(tl.float32)
pf = _round16_f32(p32, IS_BF16)
o32 = rv.to(tl.float32) + pf
_store16(out, base + offs, o32, mask, IS_BF16)
@triton.jit
def _rga_flat_m1(
out,
res,
upd,
gate,
numel,
hid: tl.constexpr,
IS_BF16: tl.constexpr,
BLK: tl.constexpr,
):
pid = tl.program_id(0).to(tl.int64)
offs = pid * BLK + tl.arange(0, BLK).to(tl.int64)
mask = offs < numel
rv = tl.load(res + offs, mask=mask, eviction_policy="evict_first")
uv = tl.load(upd + offs, mask=mask, eviction_policy="evict_first")
gv = tl.load(gate + (offs % hid), mask=mask, eviction_policy="evict_last")
p32 = uv.to(tl.float32) * gv.to(tl.float32)
pf = _round16_f32(p32, IS_BF16)
o32 = rv.to(tl.float32) + pf
_store16(out, offs, o32, mask, IS_BF16)
@triton.jit
def _rga_transposed(
out,
res,
upd,
gate,
tokens,
hid: tl.constexpr,
IS_BF16: tl.constexpr,
IS_16: tl.constexpr,
TILE: tl.constexpr,
):
# residual/out have logical [batch, tokens, hid] with physical strides (T*H, 1, T);
# update is contiguous. out[b,t,h] = res + upd * gate[h].
# h-major tile indexing ([TILE_h, TILE_t]) makes the residual/output access
# coalesced along the physical stride-1 token dim; update (read once) takes
# the strided pattern instead. Measured faster than the t-major layout.
pid_t = tl.program_id(0).to(tl.int64)
pid_h = tl.program_id(1).to(tl.int64)
pid_b = tl.program_id(2).to(tl.int64)
t = pid_t * TILE + tl.arange(0, TILE).to(tl.int64)
h = pid_h * TILE + tl.arange(0, TILE).to(tl.int64)
mt = t < tokens
mh = h < hid
gv = tl.load(gate + h, mask=mh, other=0.0)
base = pid_b * tokens * hid
m2 = mh[:, None] & mt[None, :]
roffs = h[:, None] * tokens + t[None, :]
rv = tl.load(res + base + roffs, mask=m2, other=0.0)
uv = tl.load(upd + base + t[None, :] * hid + h[:, None], mask=m2, other=0.0)
if IS_16:
p32 = uv.to(tl.float32) * gv[:, None].to(tl.float32)
pf = _round16_f32(p32, IS_BF16)
o32 = rv.to(tl.float32) + pf
if IS_BF16:
bits = tl.inline_asm_elementwise(
"cvt.rn.bf16.f32 $0, $1;",
"=h,r",
[o32],
dtype=tl.int16,
is_pure=True,
pack=1,
)
tl.store(out + base + roffs, bits.to(tl.bfloat16, bitcast=True), mask=m2)
else:
bits = tl.inline_asm_elementwise(
"cvt.rn.f16.f32 $0, $1;",
"=h,r",
[o32],
dtype=tl.int16,
is_pure=True,
pack=1,
)
tl.store(out + base + roffs, bits.to(tl.float16, bitcast=True), mask=m2)
else:
tl.store(out + base + roffs, rv + uv * gv[:, None], mask=m2)
def _residual_gate_add_triton(residual, update, gate):
if not can_use_residual_gate_add_cuda(residual, update, gate):
raise RuntimeError("unsupported input for residual_gate_add CUDA")
out = torch.empty_strided(
residual.shape, residual.stride(), dtype=residual.dtype, device=residual.device
)
is_bf16 = residual.dtype == torch.bfloat16
dtype16 = residual.dtype in (torch.float16, torch.bfloat16)
if _is_transposed_dense_residual(residual, update, gate):
tokens = residual.shape[1]
hid = residual.shape[2]
batch = residual.shape[0]
if tokens * hid * batch <= 65536:
tile, warps = 16, 4
elif tokens * hid * batch >= 1 << 20:
tile, warps = 64, 8
else:
tile, warps = 32, 8
grid = (triton.cdiv(tokens, tile), triton.cdiv(hid, tile), batch)
_rga_transposed[grid](
out,
residual,
update,
gate,
tokens,
hid,
is_bf16,
dtype16,
TILE=tile,
num_warps=warps,
)
return out
if not dtype16:
# fp32: exact fp32 math, plain Triton elementwise
numel = residual.numel()
hid = residual.shape[-1]
if gate.shape == residual.shape:
mode = 0
elif _is_row_broadcast_gate(residual, gate):
mode = 1
else:
mode = 2
if numel <= 8192:
_rga_flat[(triton.cdiv(numel, 256),)](
out,
residual,
update,
gate,
numel,
hid,
mode,
False,
False,
BLK=256,
num_warps=4,
)
else:
_rga_flat[(triton.cdiv(numel, 1024),)](
out,
residual,
update,
gate,
numel,
hid,
mode,
False,
False,
BLK=1024,
num_warps=4,
)
return out
hid = residual.shape[-1]
numel = residual.numel()
if _is_row_broadcast_gate(residual, gate):
rows = numel // hid
if hid >= 4096 or (rows <= 512 and numel >= 1 << 20):
_rga_flat_m1[(triton.cdiv(numel, 1024),)](
out, residual, update, gate, numel, hid, is_bf16, BLK=1024, num_warps=4
)
elif hid <= 16384:
blk = triton.next_power_of_2(hid)
_rga_row[(rows,)](
out,
residual,
update,
gate,
hid,
is_bf16,
hid < 4608,
hid >= 4096,
BLK=blk,
num_warps=4 if hid >= 4608 else 8,
)
else:
_rga_flat[(triton.cdiv(numel, 1024),)](
out,
residual,
update,
gate,
numel,
hid,
1,
is_bf16,
True,
BLK=1024,
num_warps=4,
)
return out
mode = 0 if gate.shape == residual.shape else 2
if numel <= 8192:
_rga_flat[(triton.cdiv(numel, 256),)](
out,
residual,
update,
gate,
numel,
hid,
mode,
is_bf16,
True,
BLK=256,
num_warps=4,
)
else:
_rga_flat[(triton.cdiv(numel, 1024),)](
out,
residual,
update,
gate,
numel,
hid,
mode,
is_bf16,
True,
BLK=1024,
num_warps=4,
)
return out
@register_custom_op(
op_name="diffusion_residual_gate_add",
mutates_args=[],
fake_impl=_fake_impl,
)
def _residual_gate_add_custom_op(
residual: torch.Tensor, update: torch.Tensor, gate: torch.Tensor
) -> torch.Tensor:
with torch.cuda.device(residual.device):
return _residual_gate_add_triton(residual, update, gate)
def _gate_mode(residual: torch.Tensor, gate: torch.Tensor) -> int:
"""0 = full, 1 = broadcast row (hidden_size), 2 = per-token (rows)."""
if gate.shape == residual.shape: