[Diffusion] Fuse lossless Wan VAE post-ops for LongLive 2 I2V (#40405)

This commit is contained in:
Xiaoyu Zhang
2026-09-24 17:40:07 +08:00
committed by GitHub
parent 261cb826e6
commit c85df5becc
6 changed files with 420 additions and 7 deletions
@@ -0,0 +1,115 @@
// SPDX-License-Identifier: Apache-2.0
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/vec.cuh>
#include <tvm/ffi/container/tensor.h>
#include <algorithm>
#include <cmath>
#include <cstdint>
namespace sglang {
/// \brief Fuse FP32 divide, scale, affine, and SiLU without changing aten's L2 reduction.
template <typename X, typename A, bool kChannelsLast>
__global__ void wan_norm_silu_post_kernel(
const X* __restrict__ input,
const float* __restrict__ denominator,
const A* __restrict__ gamma,
const A* __restrict__ bias,
float* __restrict__ output,
int64_t num_vecs,
int64_t channels,
int64_t spatial,
float scale,
bool has_bias) {
constexpr int kVec = 4;
const int64_t step = static_cast<int64_t>(blockDim.x) * gridDim.x;
for (int64_t vec = static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x; vec < num_vecs; vec += step) {
device::AlignedVector<X, kVec> x;
device::AlignedVector<float, kVec> d;
device::AlignedVector<A, kVec> g;
device::AlignedVector<A, kVec> b;
device::AlignedVector<float, kVec> result;
x.load(input, vec);
float shared_d = 0.f, shared_g = 0.f, shared_b = 0.f;
if constexpr (kChannelsLast) {
const int64_t channel = vec * kVec % channels;
shared_d = denominator[vec * kVec / channels];
g.load(gamma, channel / kVec);
if (has_bias) b.load(bias, channel / kVec);
} else {
const int64_t channel = vec * kVec / spatial % channels;
const int64_t batch = vec * kVec / (spatial * channels);
const int64_t pixel = vec * kVec % spatial;
d.load(denominator, (batch * spatial + pixel) / kVec);
shared_g = device::cast<float>(gamma[channel]);
if (has_bias) shared_b = device::cast<float>(bias[channel]);
}
#pragma unroll
for (int i = 0; i < kVec; ++i) {
const float denom = kChannelsLast ? shared_d : d[i];
const float weight = kChannelsLast ? device::cast<float>(g[i]) : shared_g;
const float offset = has_bias ? (kChannelsLast ? device::cast<float>(b[i]) : shared_b) : 0.f;
float y = __fdiv_rn(device::cast<float>(x[i]), denom);
y = __fmul_rn(y, scale);
y = __fmul_rn(y, weight);
y = __fadd_rn(y, offset);
result[i] = __fdiv_rn(y, __fadd_rn(1.f, expf(-y)));
}
result.store(output, vec);
}
}
/// \brief Launch normalization post-ops for dense NCDHW or channels-last storage.
/// \param denominator Native FP32 L2 norm, clamped to the normalization epsilon.
template <typename X, typename A, bool kChannelsLast>
void wan_norm_silu_post(
tvm::ffi::TensorView input,
tvm::ffi::TensorView denominator,
tvm::ffi::TensorView gamma,
tvm::ffi::TensorView bias,
tvm::ffi::TensorView output,
int64_t channels,
int64_t spatial,
double scale,
bool has_bias) {
using namespace host;
auto elements = SymbolicSize{"elements"};
auto pixels = SymbolicSize{"pixels"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLCUDA>();
TensorMatcher({elements}).with_dtype<X>().with_device(device_).verify(input);
TensorMatcher({pixels}).with_dtype<float>().with_device(device_).verify(denominator);
TensorMatcher({channels}).with_dtype<A>().with_device(device_).verify(gamma);
TensorMatcher({channels}).with_dtype<A>().with_device(device_).verify(bias);
TensorMatcher({elements}).with_dtype<float>().with_device(device_).verify(output);
const int64_t count = elements.unwrap();
CHECK_HOST(channels > 0 && channels % 4 == 0 && spatial > 0 && spatial % 4 == 0);
CHECK_HOST(count > 0 && count % (channels * spatial) == 0);
CHECK_HOST(pixels.unwrap() == count / channels);
constexpr int64_t threads = 256;
const auto kernel = wan_norm_silu_post_kernel<X, A, kChannelsLast>;
const auto blocks_per_sm = runtime::get_blocks_per_sm(kernel, threads);
const auto sms = runtime::get_sm_count(device_.unwrap().device_id);
const auto blocks = std::min<int64_t>(sms * blocks_per_sm, div_ceil(count / 4, threads));
LaunchKernel(blocks, threads, device_.unwrap())(
kernel,
static_cast<const X*>(input.data_ptr()),
static_cast<const float*>(denominator.data_ptr()),
static_cast<const A*>(gamma.data_ptr()),
static_cast<const A*>(bias.data_ptr()),
static_cast<float*>(output.data_ptr()),
count / 4,
channels,
spatial,
static_cast<float>(scale),
has_bias);
}
} // namespace sglang
@@ -84,6 +84,13 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
_CUDA,
"Wan VAE channels_last_3d RMSNorm + SiLU.",
),
(
"diffusion.wan_norm_silu_post",
KernelBackend.JIT,
"norm.wan_norm_silu_post:wan_norm_silu_post",
_CUDA,
"Wan VAE FP32 normalization post-ops with the native denominator.",
),
(
"diffusion.rmsnorm_scale_shift",
KernelBackend.TRITON,
@@ -580,6 +587,8 @@ _EXPORTS: dict[str, str] = {
"try_fused_scale_residual_norm_scale_shift_fp8": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
"validate_scale_shift": "norm.scale_residual_norm_cutedsl",
"try_fused_scale_residual_norm_scale_shift_nvfp4": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
"can_use_wan_norm_silu_post": "norm.wan_norm_silu_post",
"wan_norm_silu_post": "norm.wan_norm_silu_post",
"can_use_wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton",
"wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton",
"can_use_qk_rmsnorm_native": "norm.zimage_qk_rmsnorm_triton",
@@ -0,0 +1,99 @@
# SPDX-License-Identifier: Apache-2.0
"""FP32 Wan normalization post-ops with the native L2 denominator supplied."""
import torch
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
from sglang.srt.utils.custom_op import register_custom_op
@cache_once
def _module(x_dtype, affine_dtype, channels_last):
args = make_cpp_args(x_dtype, affine_dtype, channels_last)
return load_jit(
"wan_norm_silu_post",
*args,
cuda_files=["elementwise/wan_norm_silu_post.cuh"],
extra_cuda_cflags=["--fmad=false"],
cuda_wrappers=[("run", f"wan_norm_silu_post<{args}>")],
)
def can_use_wan_norm_silu_post(x, denominator, gamma, bias=None):
if (
not x.is_cuda
or torch.version.hip is not None
or torch.is_grad_enabled()
or x.requires_grad
or x.ndim != 5
or x.numel() == 0
or x.dtype not in (torch.bfloat16, torch.float32)
or x.shape[1] % 4
or (x.shape[2] * x.shape[3] * x.shape[4]) % 4
or not (
x.is_contiguous() or x.is_contiguous(memory_format=torch.channels_last_3d)
)
or x.data_ptr() % 16
or denominator.dtype != torch.float32
or denominator.device != x.device
or denominator.shape != (x.shape[0], 1, *x.shape[2:])
or not denominator.is_contiguous()
or denominator.data_ptr() % 16
or gamma.dtype not in (torch.bfloat16, torch.float32)
or gamma.device != x.device
or gamma.numel() != x.shape[1]
or not gamma.is_contiguous()
or gamma.data_ptr() % 16
):
return False
return bias is None or (
bias.device == x.device
and bias.dtype == gamma.dtype
and bias.numel() == x.shape[1]
and bias.is_contiguous()
and bias.data_ptr() % 16 == 0
)
def _storage(x, channels_last):
return x.permute(0, 2, 3, 4, 1).view(-1) if channels_last else x.view(-1)
@register_custom_op(mutates_args=["out"])
def _wan_norm_silu_post(
x: torch.Tensor,
denominator: torch.Tensor,
gamma: torch.Tensor,
bias: torch.Tensor,
out: torch.Tensor,
scale: float,
has_bias: bool,
) -> None:
channels_last = not x.is_contiguous()
_module(x.dtype, gamma.dtype, channels_last).run(
_storage(x, channels_last),
denominator.view(-1),
gamma.view(-1),
bias.view(-1),
_storage(out, channels_last),
x.shape[1],
x.shape[2] * x.shape[3] * x.shape[4],
scale,
has_bias,
)
def wan_norm_silu_post(x, denominator, gamma, bias=None, *, scale=None):
if not can_use_wan_norm_silu_post(x, denominator, gamma, bias):
raise ValueError("unsupported Wan normalization post-op inputs")
out = torch.empty_like(x, dtype=torch.float32)
_wan_norm_silu_post(
x,
denominator,
gamma,
gamma if bias is None else bias,
out,
float(x.shape[1] ** 0.5 if scale is None else scale),
bias is not None,
)
return out
@@ -6,8 +6,9 @@ Fuses every decoder ``WanRMS_norm -> SiLU`` chain into one Triton kernel on
the channels_last_3d layout. Wrappers are installed once at VAE load and
dispatch on a decode-scoped :class:`VaeFastPathGate`: ``quality="extra-high"``
and ``quality="high"`` run the fused kernel (not bitwise-identical to aten,
hence gated), while the ``"lossless"`` default runs the original module path
bit-for-bit. Install is all-or-nothing and fail-closed.
hence gated). The ``"lossless"`` path preserves aten's normalization reduction
and fuses its FP32 post-ops after first-sight exactness verification.
Install is all-or-nothing and fail-closed.
"""
import torch
@@ -24,9 +25,12 @@ logger = init_logger(__name__)
try:
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_use_nearest_upsample_nhwc,
can_use_wan_norm_silu_post,
can_use_wan_rmsnorm_silu,
nearest_upsample_nhwc,
wan_norm_silu_post,
wan_rmsnorm_silu,
)
@@ -36,10 +40,10 @@ except ImportError: # pragma: no cover
class FusedWanRMSNormSiLU(nn.Module):
"""``WanRMS_norm`` + SiLU fused via the channels_last_3d Triton kernel;
falls back to the original op chain (bit-identical to norm + ``nn.SiLU``)
for unsupported inputs and whenever the gate is off. Steps aside under
``torch.compile``, where Inductor already fuses this chain."""
"""Quality-gated full fusion or lossless FP32 post-op fusion.
Unsupported inputs and ``torch.compile`` use the original op chain.
"""
def __init__(self, norm: nn.Module, gate: VaeFastPathGate) -> None:
super().__init__()
@@ -50,12 +54,58 @@ class FusedWanRMSNormSiLU(nn.Module):
self.bias = norm.bias
self.scale = float(norm.scale)
self._sgl_gate = gate
self._post_gate = BitExactFusionGate(
"Wan norm+SiLU post-ops", per_signature=True
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
if self._sgl_gate.enabled and not torch.compiler.is_compiling():
bias = self.bias if isinstance(self.bias, torch.Tensor) else None
if can_use_wan_rmsnorm_silu(x, self.gamma, bias):
return wan_rmsnorm_silu(x, self.gamma, bias, rms_scale=self.scale)
if (
not torch.compiler.is_compiling()
and not self._post_gate.disabled
and x.is_cuda
and torch.version.hip is None
and not torch.is_grad_enabled()
and x.ndim == 5
and (
x.dtype == torch.float32
or (x.dtype == torch.bfloat16 and torch.is_autocast_enabled("cuda"))
)
and (isinstance(self.bias, torch.Tensor) or self.bias == 0.0)
):
sig = (x.device, x.dtype, x.shape, x.stride(), self.gamma.dtype)
verified = self._post_gate.is_verified(sig)
if verified or not torch.cuda.is_current_stream_capturing():
# Keep this outside the custom op: CUDA autocast promotes the
# native norm to FP32, including for BF16 VAE activations.
denominator = x.norm(p=2, dim=1, keepdim=True).clamp_min(1e-12)
bias = self.bias if isinstance(self.bias, torch.Tensor) else None
if can_use_wan_norm_silu_post(x, denominator, self.gamma, bias):
try:
out = wan_norm_silu_post(
x, denominator, self.gamma, bias, scale=self.scale
)
if verified:
return out
ref = F.silu(
(x / denominator) * self.scale * self.gamma + self.bias
)
return self._post_gate.accept_or_fallback(
out,
ref,
sig=sig,
equal=lambda a, b: torch.equal(
a.view(torch.int32), b.view(torch.int32)
),
logger=logger,
)
except Exception as exc:
self._post_gate.on_exception(exc, logger=logger)
# Unsupported layouts/dtypes still reuse the native reduction.
return F.silu((x / denominator) * self.scale * self.gamma + self.bias)
# WanRMS_norm.forward (channel-first) + SiLU, same ops in the same
# order, so the off-path stays bit-identical.
return F.silu(F.normalize(x, dim=1) * self.scale * self.gamma + self.bias)