[diffusion] kernels: take the Wan VAE decoder's remaining first-decode work off the request path (#41830)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-06 09:05:29 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 3120bb87a2
commit 5d385007b3
5 changed files with 161 additions and 23 deletions
@@ -22,11 +22,10 @@ import triton.language as tl # type: ignore
_MAX_INT32 = 2**31 - 1
# The per-request sizes only index. The channel count, ``total`` and the
# strides stay specialized for the channel vectorization, and the per-layer
# padding and cache length for their constant folding (1.5% of kernel time).
@triton.jit(do_not_specialize=["T", "H", "W", "out_t", "out_h", "out_w"])
def _cat_pad_cl3d_kernel(
_CAT_PAD_SIZES = ["T", "H", "W", "out_t", "out_h", "out_w"]
def _cat_pad_cl3d(
x_ptr,
cache_ptr,
out_ptr,
@@ -100,6 +99,17 @@ def _cat_pad_cl3d_kernel(
tl.store(keep_ptr + k_off, vals, mask=keep)
# The per-request sizes only index. The channel count, ``total`` and the
# strides stay specialized for the channel vectorization, and the per-layer
# padding and cache length for their constant folding (1.5% of kernel time).
_cat_pad_cl3d_kernel = triton.jit(do_not_specialize=_CAT_PAD_SIZES)(_cat_pad_cl3d)
# The strides of a channels-first input change with the resolution, and its
# loads gather along the channels whatever their values.
_cat_pad_cl3d_gather_kernel = triton.jit(
do_not_specialize=_CAT_PAD_SIZES + ["sxb", "sxc", "sxt", "sxh"]
)(_cat_pad_cl3d)
def cat_pad_channels_last_3d(
x: torch.Tensor,
cache_x: torch.Tensor | None,
@@ -170,10 +180,11 @@ def cat_pad_channels_last_3d(
scb, scc, sct, sch, scw = cache_x.stride()
sxb, sxc, sxt, sxh, sxw = x.stride()
kernel = _cat_pad_cl3d_kernel if sxc == 1 else _cat_pad_cl3d_gather_kernel
BLOCK = 512
grid = (triton.cdiv(total, BLOCK),)
with torch.get_device_module().device(x.device):
_cat_pad_cl3d_kernel[grid](
kernel[grid](
x,
cache_arg,
out,
@@ -210,9 +221,36 @@ def cat_pad_channels_last_3d(
return out
# ``out_w`` stays specialized: it is the contiguous dimension of the
# channels-first traversal.
@triton.jit(do_not_specialize=["out_t", "out_h", "t_offset"])
# Sizes and strides change with the resolution, so only unit strides stay
# specialized. The other strides of main and out, and out_w when W is the
# innermost dim, come in units of VEC: that tells the compiler, as their
# specialization did, that each run of VEC outputs is contiguous and aligned.
@triton.jit(
do_not_specialize=[
"out_t",
"out_h",
"out_w",
"t_offset",
# scaled by VEC in both traversals, never the unit stride
"smb",
"smt",
"smh",
"sob",
"sot",
"soh",
],
do_not_specialize_on_alignment=[
"smc",
"smw",
"soc",
"sow",
"ssb",
"ssc",
"sst",
"ssh",
"ssw",
],
)
def _dup_up3d_add_kernel(
main_ptr,
src_ptr,
@@ -242,6 +280,7 @@ def _dup_up3d_add_kernel(
FS: tl.constexpr,
REPEATS: tl.constexpr,
CHANNELS_INNER: tl.constexpr,
VEC: tl.constexpr,
IDX64: tl.constexpr,
BLOCK: tl.constexpr,
):
@@ -250,6 +289,12 @@ def _dup_up3d_add_kernel(
else:
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = offs < total
smb = smb * VEC
smt = smt * VEC
smh = smh * VEC
sob = sob * VEC
sot = sot * VEC
soh = soh * VEC
# Logical (B, C_out, out_t, out_h, out_w) index; the output tensor keeps
# ``main``'s stride order (``empty_like`` preserve), matching what the
@@ -259,6 +304,8 @@ def _dup_up3d_add_kernel(
# follows the output's memory order (channels innermost for NHWC-style
# ``main``) so stores and ``main`` loads stay coalesced.
if CHANNELS_INNER:
smw = smw * VEC
sow = sow * VEC
oc = offs % C_out
rest = offs // C_out
ow = rest % out_w
@@ -268,6 +315,9 @@ def _dup_up3d_add_kernel(
o_t = rest % out_t
ob = rest // out_t
else:
smc = smc * VEC
soc = soc * VEC
out_w = out_w * VEC
ow = offs % out_w
rest = offs // out_w
oh = rest % out_h
@@ -300,6 +350,17 @@ def _dup_up3d_add_kernel(
tl.store(out_ptr + o_off, vals, mask=mask)
# Each thread holds BLOCK / (32 * num_warps) = 4 consecutive outputs.
_MAX_VEC = 4
def _vec_width(inner: int, strides) -> int:
vec = _MAX_VEC
while vec > 1 and (inner % vec or any(s % vec for s in strides)):
vec //= 2
return vec
def dup_up3d_add(
main: torch.Tensor,
src: torch.Tensor,
@@ -353,6 +414,19 @@ def dup_up3d_add(
smb, smc, smt, smh, smw = main.stride()
ssb, ssc, sst, ssh, ssw = src.stride()
sob, soc, sot, soh, sow = out.stride()
channels_inner = out.stride(1) == 1 and exp_shape[1] > 1
out_w = exp_shape[4]
if channels_inner:
unit, inner, scaled = (smc, soc), exp_shape[1], (smw, sow)
else:
unit, inner, scaled = (smw, sow), out_w, (smc, soc)
common = (smb, smt, smh, sob, sot, soh)
vec = _vec_width(inner, common + scaled) if unit == (1, 1) else 1
smb, smt, smh, sob, sot, soh = (s // vec for s in common)
if channels_inner:
smw, sow = smw // vec, sow // vec
else:
smc, soc, out_w = smc // vec, soc // vec, out_w // vec
BLOCK = 512
grid = (triton.cdiv(total, BLOCK),)
with torch.get_device_module().device(main.device):
@@ -364,7 +438,7 @@ def dup_up3d_add(
exp_shape[1],
exp_shape[2],
exp_shape[3],
exp_shape[4],
out_w,
t_offset,
smb,
smc,
@@ -384,7 +458,8 @@ def dup_up3d_add(
FT=factor_t,
FS=factor_s,
REPEATS=repeats,
CHANNELS_INNER=out.stride(1) == 1 and exp_shape[1] > 1,
CHANNELS_INNER=channels_inner,
VEC=vec,
IDX64=total >= _MAX_INT32,
BLOCK=BLOCK,
)
@@ -76,7 +76,9 @@ class FusedWanRMSNormSiLU(nn.Module):
)
and (isinstance(self.bias, torch.Tensor) or self.bias == 0.0)
):
sig = (x.device, x.dtype, x.shape, x.stride(), self.gamma.dtype)
# Every element takes the same ops whatever the sizes, so exactness
# depends only on the dtypes and on which layout the kernel runs.
sig = (x.device, x.dtype, x.is_contiguous(), self.gamma.dtype)
verified = self._post_gate.is_verified(sig)
if verified or not torch.cuda.is_current_stream_capturing():
# Keep this outside the custom op: CUDA autocast promotes the