From 5d385007b32b42b33e4404e1969410e06472b06e Mon Sep 17 00:00:00 2001 From: Mick Date: Tue, 6 Oct 2026 09:05:29 +0800 Subject: [PATCH] [diffusion] kernels: take the Wan VAE decoder's remaining first-decode work off the request path (#41830) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- .../layout/wan_causal_cache_triton.py | 97 ++++++++++++++++--- .../runtime/models/vaes/wan_vae_cuda_opt.py | 4 +- .../kernels/ops/diffusion/test_layout.py | 33 +++++-- .../diffusion/test_shape_specialization.py | 33 ++++++- .../ops/diffusion/test_wan_norm_silu_post.py | 17 ++++ 5 files changed, 161 insertions(+), 23 deletions(-) diff --git a/python/sglang/kernels/ops/diffusion/layout/wan_causal_cache_triton.py b/python/sglang/kernels/ops/diffusion/layout/wan_causal_cache_triton.py index 4493a60259..a60949aed7 100644 --- a/python/sglang/kernels/ops/diffusion/layout/wan_causal_cache_triton.py +++ b/python/sglang/kernels/ops/diffusion/layout/wan_causal_cache_triton.py @@ -22,11 +22,10 @@ import triton.language as tl # type: ignore _MAX_INT32 = 2**31 - 1 -# The per-request sizes only index. The channel count, ``total`` and the -# strides stay specialized for the channel vectorization, and the per-layer -# padding and cache length for their constant folding (1.5% of kernel time). -@triton.jit(do_not_specialize=["T", "H", "W", "out_t", "out_h", "out_w"]) -def _cat_pad_cl3d_kernel( +_CAT_PAD_SIZES = ["T", "H", "W", "out_t", "out_h", "out_w"] + + +def _cat_pad_cl3d( x_ptr, cache_ptr, out_ptr, @@ -100,6 +99,17 @@ def _cat_pad_cl3d_kernel( tl.store(keep_ptr + k_off, vals, mask=keep) +# The per-request sizes only index. The channel count, ``total`` and the +# strides stay specialized for the channel vectorization, and the per-layer +# padding and cache length for their constant folding (1.5% of kernel time). +_cat_pad_cl3d_kernel = triton.jit(do_not_specialize=_CAT_PAD_SIZES)(_cat_pad_cl3d) +# The strides of a channels-first input change with the resolution, and its +# loads gather along the channels whatever their values. +_cat_pad_cl3d_gather_kernel = triton.jit( + do_not_specialize=_CAT_PAD_SIZES + ["sxb", "sxc", "sxt", "sxh"] +)(_cat_pad_cl3d) + + def cat_pad_channels_last_3d( x: torch.Tensor, cache_x: torch.Tensor | None, @@ -170,10 +180,11 @@ def cat_pad_channels_last_3d( scb, scc, sct, sch, scw = cache_x.stride() sxb, sxc, sxt, sxh, sxw = x.stride() + kernel = _cat_pad_cl3d_kernel if sxc == 1 else _cat_pad_cl3d_gather_kernel BLOCK = 512 grid = (triton.cdiv(total, BLOCK),) with torch.get_device_module().device(x.device): - _cat_pad_cl3d_kernel[grid]( + kernel[grid]( x, cache_arg, out, @@ -210,9 +221,36 @@ def cat_pad_channels_last_3d( return out -# ``out_w`` stays specialized: it is the contiguous dimension of the -# channels-first traversal. -@triton.jit(do_not_specialize=["out_t", "out_h", "t_offset"]) +# Sizes and strides change with the resolution, so only unit strides stay +# specialized. The other strides of main and out, and out_w when W is the +# innermost dim, come in units of VEC: that tells the compiler, as their +# specialization did, that each run of VEC outputs is contiguous and aligned. +@triton.jit( + do_not_specialize=[ + "out_t", + "out_h", + "out_w", + "t_offset", + # scaled by VEC in both traversals, never the unit stride + "smb", + "smt", + "smh", + "sob", + "sot", + "soh", + ], + do_not_specialize_on_alignment=[ + "smc", + "smw", + "soc", + "sow", + "ssb", + "ssc", + "sst", + "ssh", + "ssw", + ], +) def _dup_up3d_add_kernel( main_ptr, src_ptr, @@ -242,6 +280,7 @@ def _dup_up3d_add_kernel( FS: tl.constexpr, REPEATS: tl.constexpr, CHANNELS_INNER: tl.constexpr, + VEC: tl.constexpr, IDX64: tl.constexpr, BLOCK: tl.constexpr, ): @@ -250,6 +289,12 @@ def _dup_up3d_add_kernel( else: offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) mask = offs < total + smb = smb * VEC + smt = smt * VEC + smh = smh * VEC + sob = sob * VEC + sot = sot * VEC + soh = soh * VEC # Logical (B, C_out, out_t, out_h, out_w) index; the output tensor keeps # ``main``'s stride order (``empty_like`` preserve), matching what the @@ -259,6 +304,8 @@ def _dup_up3d_add_kernel( # follows the output's memory order (channels innermost for NHWC-style # ``main``) so stores and ``main`` loads stay coalesced. if CHANNELS_INNER: + smw = smw * VEC + sow = sow * VEC oc = offs % C_out rest = offs // C_out ow = rest % out_w @@ -268,6 +315,9 @@ def _dup_up3d_add_kernel( o_t = rest % out_t ob = rest // out_t else: + smc = smc * VEC + soc = soc * VEC + out_w = out_w * VEC ow = offs % out_w rest = offs // out_w oh = rest % out_h @@ -300,6 +350,17 @@ def _dup_up3d_add_kernel( tl.store(out_ptr + o_off, vals, mask=mask) +# Each thread holds BLOCK / (32 * num_warps) = 4 consecutive outputs. +_MAX_VEC = 4 + + +def _vec_width(inner: int, strides) -> int: + vec = _MAX_VEC + while vec > 1 and (inner % vec or any(s % vec for s in strides)): + vec //= 2 + return vec + + def dup_up3d_add( main: torch.Tensor, src: torch.Tensor, @@ -353,6 +414,19 @@ def dup_up3d_add( smb, smc, smt, smh, smw = main.stride() ssb, ssc, sst, ssh, ssw = src.stride() sob, soc, sot, soh, sow = out.stride() + channels_inner = out.stride(1) == 1 and exp_shape[1] > 1 + out_w = exp_shape[4] + if channels_inner: + unit, inner, scaled = (smc, soc), exp_shape[1], (smw, sow) + else: + unit, inner, scaled = (smw, sow), out_w, (smc, soc) + common = (smb, smt, smh, sob, sot, soh) + vec = _vec_width(inner, common + scaled) if unit == (1, 1) else 1 + smb, smt, smh, sob, sot, soh = (s // vec for s in common) + if channels_inner: + smw, sow = smw // vec, sow // vec + else: + smc, soc, out_w = smc // vec, soc // vec, out_w // vec BLOCK = 512 grid = (triton.cdiv(total, BLOCK),) with torch.get_device_module().device(main.device): @@ -364,7 +438,7 @@ def dup_up3d_add( exp_shape[1], exp_shape[2], exp_shape[3], - exp_shape[4], + out_w, t_offset, smb, smc, @@ -384,7 +458,8 @@ def dup_up3d_add( FT=factor_t, FS=factor_s, REPEATS=repeats, - CHANNELS_INNER=out.stride(1) == 1 and exp_shape[1] > 1, + CHANNELS_INNER=channels_inner, + VEC=vec, IDX64=total >= _MAX_INT32, BLOCK=BLOCK, ) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/wan_vae_cuda_opt.py b/python/sglang/multimodal_gen/runtime/models/vaes/wan_vae_cuda_opt.py index 2aec574fb9..fc558f1c36 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/wan_vae_cuda_opt.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/wan_vae_cuda_opt.py @@ -76,7 +76,9 @@ class FusedWanRMSNormSiLU(nn.Module): ) and (isinstance(self.bias, torch.Tensor) or self.bias == 0.0) ): - sig = (x.device, x.dtype, x.shape, x.stride(), self.gamma.dtype) + # Every element takes the same ops whatever the sizes, so exactness + # depends only on the dtypes and on which layout the kernel runs. + sig = (x.device, x.dtype, x.is_contiguous(), self.gamma.dtype) verified = self._post_gate.is_verified(sig) if verified or not torch.cuda.is_current_stream_capturing(): # Keep this outside the custom op: CUDA autocast promotes the diff --git a/test/registered/kernels/ops/diffusion/test_layout.py b/test/registered/kernels/ops/diffusion/test_layout.py index a12c64f58c..4e41ccdee0 100644 --- a/test/registered/kernels/ops/diffusion/test_layout.py +++ b/test/registered/kernels/ops/diffusion/test_layout.py @@ -63,6 +63,18 @@ def _cl3d(shape, dtype): ) +def _channels_first_3d(shape, dtype): + # Frames-major storage viewed as (B, C, T, H, W), like the WanResample output. + b, c, t, h, w = shape + return torch.randn((b, t, c, h, w), device=DEVICE, dtype=dtype).permute( + 0, 2, 1, 3, 4 + ) + + +def _video(layout, shape, dtype): + return _cl3d(shape, dtype) if layout == "cl" else _channels_first_3d(shape, dtype) + + # --------------------------------------------------------------------------- # USP output head merge # --------------------------------------------------------------------------- @@ -530,6 +542,7 @@ def _ref_cat_pad(x, cache, padding): @torch.no_grad() +@pytest.mark.parametrize("x_layout", ["cl", "cf"]) @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) @pytest.mark.parametrize( "c,t,h,w,cache_t,pads", @@ -541,9 +554,9 @@ def _ref_cat_pad(x, cache, padding): (48, 4, 10, 14, 2, (1, 1, 1, 1, 2, 0)), # encoder-style T=4 chunk ], ) -def test_cat_pad_channels_last_3d_bitwise(dtype, c, t, h, w, cache_t, pads): +def test_cat_pad_channels_last_3d_bitwise(dtype, c, t, h, w, cache_t, pads, x_layout): torch.cuda.manual_seed(0) - x = _cl3d((1, c, t, h, w), dtype) + x = _video(x_layout, (1, c, t, h, w), dtype) cache = None if cache_t: # Strided interior view: caches may arrive as non-contiguous slices. @@ -573,23 +586,27 @@ def test_cat_pad_channels_last_3d_bitwise(dtype, c, t, h, w, cache_t, pads): @torch.no_grad() @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) +@pytest.mark.parametrize( + "main_layout,src_layout", [("cf", "cl"), ("cf", "cf"), ("cl", "cf"), ("cl", "cl")] +) @pytest.mark.parametrize( "c_in,c_out,t,h,w,ft,fs,drop", [ (128, 64, 1, 10, 14, 2, 2, False), (128, 64, 1, 10, 14, 2, 2, True), # first_chunk slicing (64, 32, 2, 10, 14, 1, 2, False), + (128, 64, 1, 10, 7, 2, 2, False), # rows of 14: runs of 2 outputs + (32, 32, 2, 10, 7, 1, 1, False), # rows of 7: no vector runs ], ) -def test_dup_up3d_add_bitwise(dtype, c_in, c_out, t, h, w, ft, fs, drop): +def test_dup_up3d_add_bitwise( + dtype, c_in, c_out, t, h, w, ft, fs, drop, main_layout, src_layout +): torch.cuda.manual_seed(0) repeats = c_out * ft * fs * fs // c_in - src = _cl3d((1, c_in, t, h, w), dtype) + src = _video(src_layout, (1, c_in, t, h, w), dtype) t_out = t * ft - (ft - 1 if drop else 0) - # Main arm as a permuted view, like the WanResample 2D output. - main = torch.randn( - (1, t_out, c_out, h * fs, w * fs), device=DEVICE, dtype=dtype - ).permute(0, 2, 1, 3, 4) + main = _video(main_layout, (1, c_out, t_out, h * fs, w * fs), dtype) dup = src.repeat_interleave(repeats, dim=1) dup = dup.view(1, c_out, ft, fs, fs, t, h, w) diff --git a/test/registered/kernels/ops/diffusion/test_shape_specialization.py b/test/registered/kernels/ops/diffusion/test_shape_specialization.py index 9594c16dd6..7811820f55 100644 --- a/test/registered/kernels/ops/diffusion/test_shape_specialization.py +++ b/test/registered/kernels/ops/diffusion/test_shape_specialization.py @@ -6,9 +6,11 @@ with the resolution, the prompt and the input image, so a class the server's warmup did not hit compiles on the first request that does: 0.3-0.5 s per kernel on a cold cache. These kernels use such sizes only for row indices and row masks, so they take them unspecialized, at no cost to vectorization. -The Wan VAE layout kernels keep their channel count, element total and -strides specialized; with channels a multiple of 16 those stay in one class -at every resolution. +The Wan VAE layout kernels keep their channel count and element total +specialized, which stay in one class at every resolution with channels a +multiple of 16. The strides of a channels-last tensor are multiples of the +channels too; those of a channels-first one change with the resolution, so +they go unspecialized, with the vector width passed as a layout constexpr. """ import sys @@ -130,6 +132,11 @@ def _cl3d(*shape): return _bf16(*shape).contiguous(memory_format=torch.channels_last_3d) +def _cf3d(b, c, t, h, w): + # the frames-major view WanResample hands the next block + return _bf16(b, t, c, h, w).permute(0, 2, 1, 3, 4) + + # One-frame chunks are real (the Wan VAE decodes frame by frame), one-pixel # frames are not: their size-1 dims get unit strides, and the strides stay # specialized on purpose. @@ -147,6 +154,22 @@ def _wan_dup_up3d(n): dup_up3d_add(main, src, 2, 2, 4, False) +def _wan_cat_pad_channels_first(n): + # a first-chunk conv input behind a channels-first upsample + hw = max(n, 2) + cat_pad_channels_last_3d(_cf3d(1, 96, 1, hw, hw), None, (1, 1, 1, 1, 2, 0)) + + +def _wan_dup_up3d_layouts(n, main_layout, src_layout): + # even source widths keep the upsampled rows a multiple of 4 wide, as + # they are at every Wan2.2 resolution + hw = n + n % 2 + make = {"cl": _cl3d, "cf": _cf3d} + src = make[src_layout](1, 128, 1, hw, hw) + main = make[main_layout](1, 128, 2, 2 * hw, 2 * hw) + dup_up3d_add(main, src, 2, 2, 8, False) + + LAUNCHES = { "select01": lambda n: _select01(n, residual=False), "residual_select01": lambda n: _select01(n, residual=True), @@ -157,7 +180,11 @@ LAUNCHES = { "temb_table_slices": _temb_table_slices, "pack_qkv": _pack_qkv, "wan_cat_pad": _wan_cat_pad, + "wan_cat_pad_channels_first": _wan_cat_pad_channels_first, "wan_dup_up3d": _wan_dup_up3d, + "wan_dup_up3d_cf_cf": lambda n: _wan_dup_up3d_layouts(n, "cf", "cf"), + "wan_dup_up3d_cf_cl": lambda n: _wan_dup_up3d_layouts(n, "cf", "cl"), + "wan_dup_up3d_cl_cf": lambda n: _wan_dup_up3d_layouts(n, "cl", "cf"), } diff --git a/test/registered/kernels/ops/diffusion/test_wan_norm_silu_post.py b/test/registered/kernels/ops/diffusion/test_wan_norm_silu_post.py index 060765c9e2..912b600e19 100644 --- a/test/registered/kernels/ops/diffusion/test_wan_norm_silu_post.py +++ b/test/registered/kernels/ops/diffusion/test_wan_norm_silu_post.py @@ -74,6 +74,23 @@ class TestWanNormSiLUPost(CustomTestCase): ) self.assertTrue(torch.equal(original + 0.125, x)) + @torch.inference_mode() + def test_first_sight_check_runs_once_per_layout(self): + # a request at a new resolution must not pay the reference chain and + # the host sync again + module = self.module(bias=True, dtype=torch.float32) + shapes = ((1, 64, 1, 8, 12), (1, 64, 2, 10, 20), (2, 64, 3, 16, 4)) + with torch.autocast("cuda", dtype=torch.bfloat16): + for shape in shapes: + for layout in (torch.contiguous_format, torch.channels_last_3d): + x = torch.randn(shape, device="cuda").to(memory_format=layout) + self.assertBitsEqual( + module(x), + reference(x, module.gamma, module.bias, module.scale), + ) + self.assertEqual(len(module._post_gate.verified_sigs), 2) + self.assertFalse(module._post_gate.disabled) + @torch.inference_mode() def test_all_finite_bf16_and_fp32_silu_boundaries(self): bits = torch.arange(65536, device="cuda", dtype=torch.int32).to(torch.int16)