[KDA+Kimi K3] Speed up SANA-Video residual gate add on H200 (#41305)
Co-authored-by: kernel-design-agents <334018530+kernel-design-agents@users.noreply.github.com>
This commit is contained in:
co-authored by
kernel-design-agents
parent
aa7a976807
commit
0bc074db83
@@ -4,6 +4,8 @@ import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
|
||||
from sglang.kernels.kda_kernels import _cuda_source
|
||||
@@ -55,12 +57,7 @@ def _fake_impl(
|
||||
)
|
||||
|
||||
|
||||
@register_custom_op(
|
||||
op_name="diffusion_residual_gate_add",
|
||||
mutates_args=[],
|
||||
fake_impl=_fake_impl,
|
||||
)
|
||||
def _residual_gate_add_custom_op(
|
||||
def _residual_gate_add_cuda_impl(
|
||||
residual: torch.Tensor, update: torch.Tensor, gate: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
out = torch.empty_strided(
|
||||
@@ -85,6 +82,347 @@ def _residual_gate_add_custom_op(
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _round16_f32(x, IS_BF16: tl.constexpr):
|
||||
if IS_BF16:
|
||||
bits = tl.inline_asm_elementwise(
|
||||
"cvt.rn.bf16.f32 $0, $1;", "=h,r", [x], dtype=tl.int16, is_pure=True, pack=1
|
||||
)
|
||||
return bits.to(tl.bfloat16, bitcast=True).to(tl.float32)
|
||||
else:
|
||||
bits = tl.inline_asm_elementwise(
|
||||
"cvt.rn.f16.f32 $0, $1;", "=h,r", [x], dtype=tl.int16, is_pure=True, pack=1
|
||||
)
|
||||
return bits.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store16(out_ptr, offs, value_f32, mask, IS_BF16: tl.constexpr):
|
||||
if IS_BF16:
|
||||
bits = tl.inline_asm_elementwise(
|
||||
"cvt.rn.bf16.f32 $0, $1;",
|
||||
"=h,r",
|
||||
[value_f32],
|
||||
dtype=tl.int16,
|
||||
is_pure=True,
|
||||
pack=1,
|
||||
)
|
||||
tl.store(out_ptr + offs, bits.to(tl.bfloat16, bitcast=True), mask=mask)
|
||||
else:
|
||||
bits = tl.inline_asm_elementwise(
|
||||
"cvt.rn.f16.f32 $0, $1;",
|
||||
"=h,r",
|
||||
[value_f32],
|
||||
dtype=tl.int16,
|
||||
is_pure=True,
|
||||
pack=1,
|
||||
)
|
||||
tl.store(out_ptr + offs, bits.to(tl.float16, bitcast=True), mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rga_flat(
|
||||
out,
|
||||
res,
|
||||
upd,
|
||||
gate,
|
||||
numel,
|
||||
hid,
|
||||
MODE: tl.constexpr,
|
||||
IS_BF16: tl.constexpr,
|
||||
IS_16: tl.constexpr,
|
||||
BLK: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0).to(tl.int64)
|
||||
offs = pid * BLK + tl.arange(0, BLK).to(tl.int64)
|
||||
mask = offs < numel
|
||||
rv = tl.load(res + offs, mask=mask)
|
||||
uv = tl.load(upd + offs, mask=mask)
|
||||
if MODE == 1:
|
||||
gv = tl.load(gate + (offs % hid), mask=mask)
|
||||
elif MODE == 2:
|
||||
gv = tl.load(gate + (offs // hid), mask=mask)
|
||||
else:
|
||||
gv = tl.load(gate + offs, mask=mask)
|
||||
if IS_16:
|
||||
p32 = uv.to(tl.float32) * gv.to(tl.float32)
|
||||
pf = _round16_f32(p32, IS_BF16)
|
||||
o32 = rv.to(tl.float32) + pf
|
||||
_store16(out, offs, o32, mask, IS_BF16)
|
||||
else:
|
||||
p32 = uv * gv
|
||||
o32 = rv + p32
|
||||
tl.store(out + offs, o32, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rga_row(
|
||||
out,
|
||||
res,
|
||||
upd,
|
||||
gate,
|
||||
hid: tl.constexpr,
|
||||
IS_BF16: tl.constexpr,
|
||||
EVG: tl.constexpr,
|
||||
CG: tl.constexpr,
|
||||
BLK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
offs = tl.arange(0, BLK)
|
||||
mask = offs < hid
|
||||
if EVG:
|
||||
gv = tl.load(gate + offs, mask=mask, eviction_policy="evict_last")
|
||||
else:
|
||||
gv = tl.load(gate + offs, mask=mask)
|
||||
base = row * hid
|
||||
if CG:
|
||||
rv = tl.load(res + base + offs, mask=mask, cache_modifier=".cg")
|
||||
uv = tl.load(upd + base + offs, mask=mask, cache_modifier=".cg")
|
||||
else:
|
||||
rv = tl.load(res + base + offs, mask=mask, eviction_policy="evict_first")
|
||||
uv = tl.load(upd + base + offs, mask=mask, eviction_policy="evict_first")
|
||||
p32 = uv.to(tl.float32) * gv.to(tl.float32)
|
||||
pf = _round16_f32(p32, IS_BF16)
|
||||
o32 = rv.to(tl.float32) + pf
|
||||
_store16(out, base + offs, o32, mask, IS_BF16)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rga_flat_m1(
|
||||
out,
|
||||
res,
|
||||
upd,
|
||||
gate,
|
||||
numel,
|
||||
hid: tl.constexpr,
|
||||
IS_BF16: tl.constexpr,
|
||||
BLK: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0).to(tl.int64)
|
||||
offs = pid * BLK + tl.arange(0, BLK).to(tl.int64)
|
||||
mask = offs < numel
|
||||
rv = tl.load(res + offs, mask=mask, eviction_policy="evict_first")
|
||||
uv = tl.load(upd + offs, mask=mask, eviction_policy="evict_first")
|
||||
gv = tl.load(gate + (offs % hid), mask=mask, eviction_policy="evict_last")
|
||||
p32 = uv.to(tl.float32) * gv.to(tl.float32)
|
||||
pf = _round16_f32(p32, IS_BF16)
|
||||
o32 = rv.to(tl.float32) + pf
|
||||
_store16(out, offs, o32, mask, IS_BF16)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rga_transposed(
|
||||
out,
|
||||
res,
|
||||
upd,
|
||||
gate,
|
||||
tokens,
|
||||
hid: tl.constexpr,
|
||||
IS_BF16: tl.constexpr,
|
||||
IS_16: tl.constexpr,
|
||||
TILE: tl.constexpr,
|
||||
):
|
||||
# residual/out have logical [batch, tokens, hid] with physical strides (T*H, 1, T);
|
||||
# update is contiguous. out[b,t,h] = res + upd * gate[h].
|
||||
# h-major tile indexing ([TILE_h, TILE_t]) makes the residual/output access
|
||||
# coalesced along the physical stride-1 token dim; update (read once) takes
|
||||
# the strided pattern instead. Measured faster than the t-major layout.
|
||||
pid_t = tl.program_id(0).to(tl.int64)
|
||||
pid_h = tl.program_id(1).to(tl.int64)
|
||||
pid_b = tl.program_id(2).to(tl.int64)
|
||||
t = pid_t * TILE + tl.arange(0, TILE).to(tl.int64)
|
||||
h = pid_h * TILE + tl.arange(0, TILE).to(tl.int64)
|
||||
mt = t < tokens
|
||||
mh = h < hid
|
||||
gv = tl.load(gate + h, mask=mh, other=0.0)
|
||||
base = pid_b * tokens * hid
|
||||
m2 = mh[:, None] & mt[None, :]
|
||||
roffs = h[:, None] * tokens + t[None, :]
|
||||
rv = tl.load(res + base + roffs, mask=m2, other=0.0)
|
||||
uv = tl.load(upd + base + t[None, :] * hid + h[:, None], mask=m2, other=0.0)
|
||||
if IS_16:
|
||||
p32 = uv.to(tl.float32) * gv[:, None].to(tl.float32)
|
||||
pf = _round16_f32(p32, IS_BF16)
|
||||
o32 = rv.to(tl.float32) + pf
|
||||
if IS_BF16:
|
||||
bits = tl.inline_asm_elementwise(
|
||||
"cvt.rn.bf16.f32 $0, $1;",
|
||||
"=h,r",
|
||||
[o32],
|
||||
dtype=tl.int16,
|
||||
is_pure=True,
|
||||
pack=1,
|
||||
)
|
||||
tl.store(out + base + roffs, bits.to(tl.bfloat16, bitcast=True), mask=m2)
|
||||
else:
|
||||
bits = tl.inline_asm_elementwise(
|
||||
"cvt.rn.f16.f32 $0, $1;",
|
||||
"=h,r",
|
||||
[o32],
|
||||
dtype=tl.int16,
|
||||
is_pure=True,
|
||||
pack=1,
|
||||
)
|
||||
tl.store(out + base + roffs, bits.to(tl.float16, bitcast=True), mask=m2)
|
||||
else:
|
||||
tl.store(out + base + roffs, rv + uv * gv[:, None], mask=m2)
|
||||
|
||||
|
||||
def _residual_gate_add_triton(residual, update, gate):
|
||||
if not can_use_residual_gate_add_cuda(residual, update, gate):
|
||||
raise RuntimeError("unsupported input for residual_gate_add CUDA")
|
||||
out = torch.empty_strided(
|
||||
residual.shape, residual.stride(), dtype=residual.dtype, device=residual.device
|
||||
)
|
||||
is_bf16 = residual.dtype == torch.bfloat16
|
||||
dtype16 = residual.dtype in (torch.float16, torch.bfloat16)
|
||||
|
||||
if _is_transposed_dense_residual(residual, update, gate):
|
||||
tokens = residual.shape[1]
|
||||
hid = residual.shape[2]
|
||||
batch = residual.shape[0]
|
||||
if tokens * hid * batch <= 65536:
|
||||
tile, warps = 16, 4
|
||||
elif tokens * hid * batch >= 1 << 20:
|
||||
tile, warps = 64, 8
|
||||
else:
|
||||
tile, warps = 32, 8
|
||||
grid = (triton.cdiv(tokens, tile), triton.cdiv(hid, tile), batch)
|
||||
_rga_transposed[grid](
|
||||
out,
|
||||
residual,
|
||||
update,
|
||||
gate,
|
||||
tokens,
|
||||
hid,
|
||||
is_bf16,
|
||||
dtype16,
|
||||
TILE=tile,
|
||||
num_warps=warps,
|
||||
)
|
||||
return out
|
||||
|
||||
if not dtype16:
|
||||
# fp32: exact fp32 math, plain Triton elementwise
|
||||
numel = residual.numel()
|
||||
hid = residual.shape[-1]
|
||||
if gate.shape == residual.shape:
|
||||
mode = 0
|
||||
elif _is_row_broadcast_gate(residual, gate):
|
||||
mode = 1
|
||||
else:
|
||||
mode = 2
|
||||
if numel <= 8192:
|
||||
_rga_flat[(triton.cdiv(numel, 256),)](
|
||||
out,
|
||||
residual,
|
||||
update,
|
||||
gate,
|
||||
numel,
|
||||
hid,
|
||||
mode,
|
||||
False,
|
||||
False,
|
||||
BLK=256,
|
||||
num_warps=4,
|
||||
)
|
||||
else:
|
||||
_rga_flat[(triton.cdiv(numel, 1024),)](
|
||||
out,
|
||||
residual,
|
||||
update,
|
||||
gate,
|
||||
numel,
|
||||
hid,
|
||||
mode,
|
||||
False,
|
||||
False,
|
||||
BLK=1024,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
hid = residual.shape[-1]
|
||||
numel = residual.numel()
|
||||
if _is_row_broadcast_gate(residual, gate):
|
||||
rows = numel // hid
|
||||
if hid >= 4096 or (rows <= 512 and numel >= 1 << 20):
|
||||
_rga_flat_m1[(triton.cdiv(numel, 1024),)](
|
||||
out, residual, update, gate, numel, hid, is_bf16, BLK=1024, num_warps=4
|
||||
)
|
||||
elif hid <= 16384:
|
||||
blk = triton.next_power_of_2(hid)
|
||||
_rga_row[(rows,)](
|
||||
out,
|
||||
residual,
|
||||
update,
|
||||
gate,
|
||||
hid,
|
||||
is_bf16,
|
||||
hid < 4608,
|
||||
hid >= 4096,
|
||||
BLK=blk,
|
||||
num_warps=4 if hid >= 4608 else 8,
|
||||
)
|
||||
else:
|
||||
_rga_flat[(triton.cdiv(numel, 1024),)](
|
||||
out,
|
||||
residual,
|
||||
update,
|
||||
gate,
|
||||
numel,
|
||||
hid,
|
||||
1,
|
||||
is_bf16,
|
||||
True,
|
||||
BLK=1024,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
mode = 0 if gate.shape == residual.shape else 2
|
||||
if numel <= 8192:
|
||||
_rga_flat[(triton.cdiv(numel, 256),)](
|
||||
out,
|
||||
residual,
|
||||
update,
|
||||
gate,
|
||||
numel,
|
||||
hid,
|
||||
mode,
|
||||
is_bf16,
|
||||
True,
|
||||
BLK=256,
|
||||
num_warps=4,
|
||||
)
|
||||
else:
|
||||
_rga_flat[(triton.cdiv(numel, 1024),)](
|
||||
out,
|
||||
residual,
|
||||
update,
|
||||
gate,
|
||||
numel,
|
||||
hid,
|
||||
mode,
|
||||
is_bf16,
|
||||
True,
|
||||
BLK=1024,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@register_custom_op(
|
||||
op_name="diffusion_residual_gate_add",
|
||||
mutates_args=[],
|
||||
fake_impl=_fake_impl,
|
||||
)
|
||||
def _residual_gate_add_custom_op(
|
||||
residual: torch.Tensor, update: torch.Tensor, gate: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
with torch.cuda.device(residual.device):
|
||||
return _residual_gate_add_triton(residual, update, gate)
|
||||
|
||||
|
||||
def _gate_mode(residual: torch.Tensor, gate: torch.Tensor) -> int:
|
||||
"""0 = full, 1 = broadcast row (hidden_size), 2 = per-token (rows)."""
|
||||
if gate.shape == residual.shape:
|
||||
|
||||
Reference in New Issue
Block a user