diff --git a/docs/docs/sglang-diffusion/fused_kernels.mdx b/docs/docs/sglang-diffusion/fused_kernels.mdx index f5daf6dc86..e8edf5e319 100644 --- a/docs/docs/sglang-diffusion/fused_kernels.mdx +++ b/docs/docs/sglang-diffusion/fused_kernels.mdx @@ -112,6 +112,7 @@ These fusion families mount under both `quality="extra-high"` and | `apply_group_norm_silu` | Triton | close | `GroupNorm + SiLU`, NCHW-contiguous | | `group_norm_silu_4d`, `group_norm_silu_rows` | Triton | close | channels-last GroupNorm(+SiLU); what lets a VAE decoder run channels_last end to end with no `nchwToNhwc` transposes | | `wan_rmsnorm_silu` | Triton | close | Wan VAE `channels_last_3d` RMSNorm + SiLU | +| `wan_norm_silu_post` | JIT CUDA | bit-exact, first-sight verified | Wan VAE FP32 divide, scale, affine, and SiLU after the unchanged native L2 reduction; dense NCDHW and `channels_last_3d` | ### adaLN modulation and gating @@ -224,7 +225,7 @@ Kernels are written against a specific eager chain in a specific model, so cover | Sana | LN+modulate, GLUMB bias+SiLU / bias+GLU, residual-gate add | | SANA-Video | Packed QKV/KV; paired fp64 interleaved RoPE; LN+modulate, GLUMB bias+SiLU / bias+GLU, and residual-gate add during BCG; BF16-input linear attention at `quality=extra-high` or `quality=high` | | Sana-WM | bidirectional gated delta-net, fused QK inverse-RMS | -| Wan | temb table slices; VAE cat+pad and DupUp3D add, `channels_last_3d` RMSNorm+SiLU | +| Wan / LongLive 2 VAE | temb table slices (Wan); VAE cat+pad and DupUp3D add; lossless FP32 normalization post-ops; full `channels_last_3d` RMSNorm+SiLU at `quality=extra-high` or `quality=high` | | Cosmos3 / Krea2 / MiniMax-H3 | QK-norm + RoPE (Krea2 also CuTe-DSL norm+scale/shift; MiniMax-H3 also indexed modulation) | | FLUX.2 VAE / HunyuanVAE / latent upsampler | GroupNorm + SiLU (channels-last two-pass for FLUX.2) | diff --git a/python/sglang/kernels/jit/csrc/elementwise/wan_norm_silu_post.cuh b/python/sglang/kernels/jit/csrc/elementwise/wan_norm_silu_post.cuh new file mode 100644 index 0000000000..40fe97efca --- /dev/null +++ b/python/sglang/kernels/jit/csrc/elementwise/wan_norm_silu_post.cuh @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: Apache-2.0 +#include +#include + +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace sglang { + +/// \brief Fuse FP32 divide, scale, affine, and SiLU without changing aten's L2 reduction. +template +__global__ void wan_norm_silu_post_kernel( + const X* __restrict__ input, + const float* __restrict__ denominator, + const A* __restrict__ gamma, + const A* __restrict__ bias, + float* __restrict__ output, + int64_t num_vecs, + int64_t channels, + int64_t spatial, + float scale, + bool has_bias) { + constexpr int kVec = 4; + const int64_t step = static_cast(blockDim.x) * gridDim.x; + for (int64_t vec = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; vec < num_vecs; vec += step) { + device::AlignedVector x; + device::AlignedVector d; + device::AlignedVector g; + device::AlignedVector b; + device::AlignedVector result; + x.load(input, vec); + float shared_d = 0.f, shared_g = 0.f, shared_b = 0.f; + if constexpr (kChannelsLast) { + const int64_t channel = vec * kVec % channels; + shared_d = denominator[vec * kVec / channels]; + g.load(gamma, channel / kVec); + if (has_bias) b.load(bias, channel / kVec); + } else { + const int64_t channel = vec * kVec / spatial % channels; + const int64_t batch = vec * kVec / (spatial * channels); + const int64_t pixel = vec * kVec % spatial; + d.load(denominator, (batch * spatial + pixel) / kVec); + shared_g = device::cast(gamma[channel]); + if (has_bias) shared_b = device::cast(bias[channel]); + } +#pragma unroll + for (int i = 0; i < kVec; ++i) { + const float denom = kChannelsLast ? shared_d : d[i]; + const float weight = kChannelsLast ? device::cast(g[i]) : shared_g; + const float offset = has_bias ? (kChannelsLast ? device::cast(b[i]) : shared_b) : 0.f; + float y = __fdiv_rn(device::cast(x[i]), denom); + y = __fmul_rn(y, scale); + y = __fmul_rn(y, weight); + y = __fadd_rn(y, offset); + result[i] = __fdiv_rn(y, __fadd_rn(1.f, expf(-y))); + } + result.store(output, vec); + } +} + +/// \brief Launch normalization post-ops for dense NCDHW or channels-last storage. +/// \param denominator Native FP32 L2 norm, clamped to the normalization epsilon. +template +void wan_norm_silu_post( + tvm::ffi::TensorView input, + tvm::ffi::TensorView denominator, + tvm::ffi::TensorView gamma, + tvm::ffi::TensorView bias, + tvm::ffi::TensorView output, + int64_t channels, + int64_t spatial, + double scale, + bool has_bias) { + using namespace host; + auto elements = SymbolicSize{"elements"}; + auto pixels = SymbolicSize{"pixels"}; + auto device_ = SymbolicDevice{}; + device_.set_options(); + TensorMatcher({elements}).with_dtype().with_device(device_).verify(input); + TensorMatcher({pixels}).with_dtype().with_device(device_).verify(denominator); + TensorMatcher({channels}).with_dtype().with_device(device_).verify(gamma); + TensorMatcher({channels}).with_dtype().with_device(device_).verify(bias); + TensorMatcher({elements}).with_dtype().with_device(device_).verify(output); + const int64_t count = elements.unwrap(); + CHECK_HOST(channels > 0 && channels % 4 == 0 && spatial > 0 && spatial % 4 == 0); + CHECK_HOST(count > 0 && count % (channels * spatial) == 0); + CHECK_HOST(pixels.unwrap() == count / channels); + constexpr int64_t threads = 256; + const auto kernel = wan_norm_silu_post_kernel; + const auto blocks_per_sm = runtime::get_blocks_per_sm(kernel, threads); + const auto sms = runtime::get_sm_count(device_.unwrap().device_id); + const auto blocks = std::min(sms * blocks_per_sm, div_ceil(count / 4, threads)); + LaunchKernel(blocks, threads, device_.unwrap())( + kernel, + static_cast(input.data_ptr()), + static_cast(denominator.data_ptr()), + static_cast(gamma.data_ptr()), + static_cast(bias.data_ptr()), + static_cast(output.data_ptr()), + count / 4, + channels, + spatial, + static_cast(scale), + has_bias); +} + +} // namespace sglang diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index 9e440ed888..4a0758616e 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -84,6 +84,13 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = ( _CUDA, "Wan VAE channels_last_3d RMSNorm + SiLU.", ), + ( + "diffusion.wan_norm_silu_post", + KernelBackend.JIT, + "norm.wan_norm_silu_post:wan_norm_silu_post", + _CUDA, + "Wan VAE FP32 normalization post-ops with the native denominator.", + ), ( "diffusion.rmsnorm_scale_shift", KernelBackend.TRITON, @@ -580,6 +587,8 @@ _EXPORTS: dict[str, str] = { "try_fused_scale_residual_norm_scale_shift_fp8": "sglang.kernels.kda_kernels.norm_scale_shift_jit", "validate_scale_shift": "norm.scale_residual_norm_cutedsl", "try_fused_scale_residual_norm_scale_shift_nvfp4": "sglang.kernels.kda_kernels.norm_scale_shift_jit", + "can_use_wan_norm_silu_post": "norm.wan_norm_silu_post", + "wan_norm_silu_post": "norm.wan_norm_silu_post", "can_use_wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton", "wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton", "can_use_qk_rmsnorm_native": "norm.zimage_qk_rmsnorm_triton", diff --git a/python/sglang/kernels/ops/diffusion/norm/wan_norm_silu_post.py b/python/sglang/kernels/ops/diffusion/norm/wan_norm_silu_post.py new file mode 100644 index 0000000000..d123f6c51d --- /dev/null +++ b/python/sglang/kernels/ops/diffusion/norm/wan_norm_silu_post.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 +"""FP32 Wan normalization post-ops with the native L2 denominator supplied.""" + +import torch + +from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args +from sglang.srt.utils.custom_op import register_custom_op + + +@cache_once +def _module(x_dtype, affine_dtype, channels_last): + args = make_cpp_args(x_dtype, affine_dtype, channels_last) + return load_jit( + "wan_norm_silu_post", + *args, + cuda_files=["elementwise/wan_norm_silu_post.cuh"], + extra_cuda_cflags=["--fmad=false"], + cuda_wrappers=[("run", f"wan_norm_silu_post<{args}>")], + ) + + +def can_use_wan_norm_silu_post(x, denominator, gamma, bias=None): + if ( + not x.is_cuda + or torch.version.hip is not None + or torch.is_grad_enabled() + or x.requires_grad + or x.ndim != 5 + or x.numel() == 0 + or x.dtype not in (torch.bfloat16, torch.float32) + or x.shape[1] % 4 + or (x.shape[2] * x.shape[3] * x.shape[4]) % 4 + or not ( + x.is_contiguous() or x.is_contiguous(memory_format=torch.channels_last_3d) + ) + or x.data_ptr() % 16 + or denominator.dtype != torch.float32 + or denominator.device != x.device + or denominator.shape != (x.shape[0], 1, *x.shape[2:]) + or not denominator.is_contiguous() + or denominator.data_ptr() % 16 + or gamma.dtype not in (torch.bfloat16, torch.float32) + or gamma.device != x.device + or gamma.numel() != x.shape[1] + or not gamma.is_contiguous() + or gamma.data_ptr() % 16 + ): + return False + return bias is None or ( + bias.device == x.device + and bias.dtype == gamma.dtype + and bias.numel() == x.shape[1] + and bias.is_contiguous() + and bias.data_ptr() % 16 == 0 + ) + + +def _storage(x, channels_last): + return x.permute(0, 2, 3, 4, 1).view(-1) if channels_last else x.view(-1) + + +@register_custom_op(mutates_args=["out"]) +def _wan_norm_silu_post( + x: torch.Tensor, + denominator: torch.Tensor, + gamma: torch.Tensor, + bias: torch.Tensor, + out: torch.Tensor, + scale: float, + has_bias: bool, +) -> None: + channels_last = not x.is_contiguous() + _module(x.dtype, gamma.dtype, channels_last).run( + _storage(x, channels_last), + denominator.view(-1), + gamma.view(-1), + bias.view(-1), + _storage(out, channels_last), + x.shape[1], + x.shape[2] * x.shape[3] * x.shape[4], + scale, + has_bias, + ) + + +def wan_norm_silu_post(x, denominator, gamma, bias=None, *, scale=None): + if not can_use_wan_norm_silu_post(x, denominator, gamma, bias): + raise ValueError("unsupported Wan normalization post-op inputs") + out = torch.empty_like(x, dtype=torch.float32) + _wan_norm_silu_post( + x, + denominator, + gamma, + gamma if bias is None else bias, + out, + float(x.shape[1] ** 0.5 if scale is None else scale), + bias is not None, + ) + return out diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/wan_vae_cuda_opt.py b/python/sglang/multimodal_gen/runtime/models/vaes/wan_vae_cuda_opt.py index 379d9aa43c..2aec574fb9 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/wan_vae_cuda_opt.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/wan_vae_cuda_opt.py @@ -6,8 +6,9 @@ Fuses every decoder ``WanRMS_norm -> SiLU`` chain into one Triton kernel on the channels_last_3d layout. Wrappers are installed once at VAE load and dispatch on a decode-scoped :class:`VaeFastPathGate`: ``quality="extra-high"`` and ``quality="high"`` run the fused kernel (not bitwise-identical to aten, -hence gated), while the ``"lossless"`` default runs the original module path -bit-for-bit. Install is all-or-nothing and fail-closed. +hence gated). The ``"lossless"`` path preserves aten's normalization reduction +and fuses its FP32 post-ops after first-sight exactness verification. +Install is all-or-nothing and fail-closed. """ import torch @@ -24,9 +25,12 @@ logger = init_logger(__name__) try: from sglang.kernels.ops.diffusion import ( + BitExactFusionGate, can_use_nearest_upsample_nhwc, + can_use_wan_norm_silu_post, can_use_wan_rmsnorm_silu, nearest_upsample_nhwc, + wan_norm_silu_post, wan_rmsnorm_silu, ) @@ -36,10 +40,10 @@ except ImportError: # pragma: no cover class FusedWanRMSNormSiLU(nn.Module): - """``WanRMS_norm`` + SiLU fused via the channels_last_3d Triton kernel; - falls back to the original op chain (bit-identical to norm + ``nn.SiLU``) - for unsupported inputs and whenever the gate is off. Steps aside under - ``torch.compile``, where Inductor already fuses this chain.""" + """Quality-gated full fusion or lossless FP32 post-op fusion. + + Unsupported inputs and ``torch.compile`` use the original op chain. + """ def __init__(self, norm: nn.Module, gate: VaeFastPathGate) -> None: super().__init__() @@ -50,12 +54,58 @@ class FusedWanRMSNormSiLU(nn.Module): self.bias = norm.bias self.scale = float(norm.scale) self._sgl_gate = gate + self._post_gate = BitExactFusionGate( + "Wan norm+SiLU post-ops", per_signature=True + ) def forward(self, x: torch.Tensor) -> torch.Tensor: if self._sgl_gate.enabled and not torch.compiler.is_compiling(): bias = self.bias if isinstance(self.bias, torch.Tensor) else None if can_use_wan_rmsnorm_silu(x, self.gamma, bias): return wan_rmsnorm_silu(x, self.gamma, bias, rms_scale=self.scale) + if ( + not torch.compiler.is_compiling() + and not self._post_gate.disabled + and x.is_cuda + and torch.version.hip is None + and not torch.is_grad_enabled() + and x.ndim == 5 + and ( + x.dtype == torch.float32 + or (x.dtype == torch.bfloat16 and torch.is_autocast_enabled("cuda")) + ) + and (isinstance(self.bias, torch.Tensor) or self.bias == 0.0) + ): + sig = (x.device, x.dtype, x.shape, x.stride(), self.gamma.dtype) + verified = self._post_gate.is_verified(sig) + if verified or not torch.cuda.is_current_stream_capturing(): + # Keep this outside the custom op: CUDA autocast promotes the + # native norm to FP32, including for BF16 VAE activations. + denominator = x.norm(p=2, dim=1, keepdim=True).clamp_min(1e-12) + bias = self.bias if isinstance(self.bias, torch.Tensor) else None + if can_use_wan_norm_silu_post(x, denominator, self.gamma, bias): + try: + out = wan_norm_silu_post( + x, denominator, self.gamma, bias, scale=self.scale + ) + if verified: + return out + ref = F.silu( + (x / denominator) * self.scale * self.gamma + self.bias + ) + return self._post_gate.accept_or_fallback( + out, + ref, + sig=sig, + equal=lambda a, b: torch.equal( + a.view(torch.int32), b.view(torch.int32) + ), + logger=logger, + ) + except Exception as exc: + self._post_gate.on_exception(exc, logger=logger) + # Unsupported layouts/dtypes still reuse the native reduction. + return F.silu((x / denominator) * self.scale * self.gamma + self.bias) # WanRMS_norm.forward (channel-first) + SiLU, same ops in the same # order, so the off-path stays bit-identical. return F.silu(F.normalize(x, dim=1) * self.scale * self.gamma + self.bias) diff --git a/test/registered/kernels/ops/diffusion/test_wan_norm_silu_post.py b/test/registered/kernels/ops/diffusion/test_wan_norm_silu_post.py new file mode 100644 index 0000000000..060765c9e2 --- /dev/null +++ b/test/registered/kernels/ops/diffusion/test_wan_norm_silu_post.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Lossless Wan normalization post-ops, dtype boundaries, and CUDA replay.""" + +import unittest +from types import SimpleNamespace + +import torch +import torch.nn.functional as F + +import sglang.multimodal_gen.runtime.models.vaes.wan_vae_cuda_opt as wan +from sglang.kernels.ops.diffusion import wan_norm_silu_post +from sglang.multimodal_gen.runtime.models.vaes.fast_path_gate import VaeFastPathGate +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=45, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +def reference(x, gamma, bias, scale): + return F.silu(F.normalize(x, dim=1) * scale * gamma + bias) + + +class TestWanNormSiLUPost(CustomTestCase): + def setUp(self): + super().setUp() + if not torch.cuda.is_available() or torch.version.hip is not None: + self.skipTest("NVIDIA CUDA required") + torch.manual_seed(1729) + + def assertBitsEqual(self, actual, expected): + self.assertEqual(actual.dtype, expected.dtype) + self.assertEqual(actual.stride(), expected.stride()) + self.assertTrue( + torch.equal(actual.view(torch.int32), expected.view(torch.int32)) + ) + + @staticmethod + def module(channels=64, *, bias=False, dtype=torch.bfloat16): + gamma = torch.nn.Parameter( + torch.randn(channels, 1, 1, 1, device="cuda", dtype=dtype) + ) + offset = torch.nn.Parameter(torch.randn_like(gamma)) if bias else 0.0 + norm = SimpleNamespace(gamma=gamma, bias=offset, scale=channels**0.5) + return wan.FusedWanRMSNormSiLU(norm, VaeFastPathGate()) + + @torch.inference_mode() + def test_autocast_promotion_layout_and_affine(self): + for dtype in (torch.bfloat16, torch.float32): + for affine in (torch.bfloat16, torch.float32): + for layout in (torch.contiguous_format, torch.channels_last_3d): + for bias in (False, True): + with self.subTest( + dtype=dtype, affine=affine, layout=layout, bias=bias + ): + module = self.module(bias=bias, dtype=affine) + x = torch.randn( + 2, 64, 3, 8, 12, device="cuda", dtype=dtype + ).to(memory_format=layout) + original = x.clone() + with torch.autocast("cuda", dtype=torch.bfloat16): + expected = reference( + x, module.gamma, module.bias, module.scale + ) + actual = module(x) + self.assertBitsEqual(actual, expected) + self.assertTrue(module._post_gate.verified) + self.assertFalse(module._post_gate.disabled) + x.add_(0.125) + self.assertBitsEqual( + module(x), + reference( + x, module.gamma, module.bias, module.scale + ), + ) + self.assertTrue(torch.equal(original + 0.125, x)) + + @torch.inference_mode() + def test_all_finite_bf16_and_fp32_silu_boundaries(self): + bits = torch.arange(65536, device="cuda", dtype=torch.int32).to(torch.int16) + values = bits.view(torch.bfloat16) + finite = values[torch.isfinite(values)].reshape(1, 64, 1, 1, 1020) + fp32 = ( + torch.tensor( + [ + -110.0, + -104.0, + -100.0, + -90.0, + -88.0, + -80.0, + -20.0, + -1.0, + -0.0, + 0.0, + 1e-40, + -1e-40, + 1.0, + 20.0, + 80.0, + 100.0, + ], + device="cuda", + ) + .repeat(64) + .reshape(1, 64, 1, 1, 16) + ) + for domain in (finite, fp32): + for layout in (torch.contiguous_format, torch.channels_last_3d): + x = domain.to(memory_format=layout) + denominator = torch.ones((1, 1, *x.shape[2:]), device="cuda") + gamma = torch.ones((64, 1, 1, 1), device="cuda", dtype=torch.bfloat16) + expected = F.silu((x / denominator) * 1.0 * gamma + 0.0) + self.assertBitsEqual( + wan_norm_silu_post(x, denominator, gamma, scale=1.0), expected + ) + + @torch.inference_mode() + def test_graph_replay_recomputes_norm_and_affine(self): + for layout in (torch.contiguous_format, torch.channels_last_3d): + module = self.module(bias=True) + x = torch.randn(2, 64, 3, 8, 12, device="cuda", dtype=torch.bfloat16).to( + memory_format=layout + ) + with torch.autocast("cuda", dtype=torch.bfloat16): + module(x) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual = module(x) + x.mul_(0.5) + module.gamma.add_(0.25) + module.bias.sub_(0.125) + graph.replay() + self.assertBitsEqual( + actual, reference(x, module.gamma, module.bias, module.scale) + ) + + +if __name__ == "__main__": + unittest.main()