[Diffusion] Fuse lossless Wan VAE post-ops for LongLive 2 I2V (#40405)
This commit is contained in:
@@ -0,0 +1,115 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
#include <sgl_kernel/vec.cuh>
|
||||
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
/// \brief Fuse FP32 divide, scale, affine, and SiLU without changing aten's L2 reduction.
|
||||
template <typename X, typename A, bool kChannelsLast>
|
||||
__global__ void wan_norm_silu_post_kernel(
|
||||
const X* __restrict__ input,
|
||||
const float* __restrict__ denominator,
|
||||
const A* __restrict__ gamma,
|
||||
const A* __restrict__ bias,
|
||||
float* __restrict__ output,
|
||||
int64_t num_vecs,
|
||||
int64_t channels,
|
||||
int64_t spatial,
|
||||
float scale,
|
||||
bool has_bias) {
|
||||
constexpr int kVec = 4;
|
||||
const int64_t step = static_cast<int64_t>(blockDim.x) * gridDim.x;
|
||||
for (int64_t vec = static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x; vec < num_vecs; vec += step) {
|
||||
device::AlignedVector<X, kVec> x;
|
||||
device::AlignedVector<float, kVec> d;
|
||||
device::AlignedVector<A, kVec> g;
|
||||
device::AlignedVector<A, kVec> b;
|
||||
device::AlignedVector<float, kVec> result;
|
||||
x.load(input, vec);
|
||||
float shared_d = 0.f, shared_g = 0.f, shared_b = 0.f;
|
||||
if constexpr (kChannelsLast) {
|
||||
const int64_t channel = vec * kVec % channels;
|
||||
shared_d = denominator[vec * kVec / channels];
|
||||
g.load(gamma, channel / kVec);
|
||||
if (has_bias) b.load(bias, channel / kVec);
|
||||
} else {
|
||||
const int64_t channel = vec * kVec / spatial % channels;
|
||||
const int64_t batch = vec * kVec / (spatial * channels);
|
||||
const int64_t pixel = vec * kVec % spatial;
|
||||
d.load(denominator, (batch * spatial + pixel) / kVec);
|
||||
shared_g = device::cast<float>(gamma[channel]);
|
||||
if (has_bias) shared_b = device::cast<float>(bias[channel]);
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kVec; ++i) {
|
||||
const float denom = kChannelsLast ? shared_d : d[i];
|
||||
const float weight = kChannelsLast ? device::cast<float>(g[i]) : shared_g;
|
||||
const float offset = has_bias ? (kChannelsLast ? device::cast<float>(b[i]) : shared_b) : 0.f;
|
||||
float y = __fdiv_rn(device::cast<float>(x[i]), denom);
|
||||
y = __fmul_rn(y, scale);
|
||||
y = __fmul_rn(y, weight);
|
||||
y = __fadd_rn(y, offset);
|
||||
result[i] = __fdiv_rn(y, __fadd_rn(1.f, expf(-y)));
|
||||
}
|
||||
result.store(output, vec);
|
||||
}
|
||||
}
|
||||
|
||||
/// \brief Launch normalization post-ops for dense NCDHW or channels-last storage.
|
||||
/// \param denominator Native FP32 L2 norm, clamped to the normalization epsilon.
|
||||
template <typename X, typename A, bool kChannelsLast>
|
||||
void wan_norm_silu_post(
|
||||
tvm::ffi::TensorView input,
|
||||
tvm::ffi::TensorView denominator,
|
||||
tvm::ffi::TensorView gamma,
|
||||
tvm::ffi::TensorView bias,
|
||||
tvm::ffi::TensorView output,
|
||||
int64_t channels,
|
||||
int64_t spatial,
|
||||
double scale,
|
||||
bool has_bias) {
|
||||
using namespace host;
|
||||
auto elements = SymbolicSize{"elements"};
|
||||
auto pixels = SymbolicSize{"pixels"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLCUDA>();
|
||||
TensorMatcher({elements}).with_dtype<X>().with_device(device_).verify(input);
|
||||
TensorMatcher({pixels}).with_dtype<float>().with_device(device_).verify(denominator);
|
||||
TensorMatcher({channels}).with_dtype<A>().with_device(device_).verify(gamma);
|
||||
TensorMatcher({channels}).with_dtype<A>().with_device(device_).verify(bias);
|
||||
TensorMatcher({elements}).with_dtype<float>().with_device(device_).verify(output);
|
||||
const int64_t count = elements.unwrap();
|
||||
CHECK_HOST(channels > 0 && channels % 4 == 0 && spatial > 0 && spatial % 4 == 0);
|
||||
CHECK_HOST(count > 0 && count % (channels * spatial) == 0);
|
||||
CHECK_HOST(pixels.unwrap() == count / channels);
|
||||
constexpr int64_t threads = 256;
|
||||
const auto kernel = wan_norm_silu_post_kernel<X, A, kChannelsLast>;
|
||||
const auto blocks_per_sm = runtime::get_blocks_per_sm(kernel, threads);
|
||||
const auto sms = runtime::get_sm_count(device_.unwrap().device_id);
|
||||
const auto blocks = std::min<int64_t>(sms * blocks_per_sm, div_ceil(count / 4, threads));
|
||||
LaunchKernel(blocks, threads, device_.unwrap())(
|
||||
kernel,
|
||||
static_cast<const X*>(input.data_ptr()),
|
||||
static_cast<const float*>(denominator.data_ptr()),
|
||||
static_cast<const A*>(gamma.data_ptr()),
|
||||
static_cast<const A*>(bias.data_ptr()),
|
||||
static_cast<float*>(output.data_ptr()),
|
||||
count / 4,
|
||||
channels,
|
||||
spatial,
|
||||
static_cast<float>(scale),
|
||||
has_bias);
|
||||
}
|
||||
|
||||
} // namespace sglang
|
||||
@@ -84,6 +84,13 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
|
||||
_CUDA,
|
||||
"Wan VAE channels_last_3d RMSNorm + SiLU.",
|
||||
),
|
||||
(
|
||||
"diffusion.wan_norm_silu_post",
|
||||
KernelBackend.JIT,
|
||||
"norm.wan_norm_silu_post:wan_norm_silu_post",
|
||||
_CUDA,
|
||||
"Wan VAE FP32 normalization post-ops with the native denominator.",
|
||||
),
|
||||
(
|
||||
"diffusion.rmsnorm_scale_shift",
|
||||
KernelBackend.TRITON,
|
||||
@@ -580,6 +587,8 @@ _EXPORTS: dict[str, str] = {
|
||||
"try_fused_scale_residual_norm_scale_shift_fp8": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
|
||||
"validate_scale_shift": "norm.scale_residual_norm_cutedsl",
|
||||
"try_fused_scale_residual_norm_scale_shift_nvfp4": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
|
||||
"can_use_wan_norm_silu_post": "norm.wan_norm_silu_post",
|
||||
"wan_norm_silu_post": "norm.wan_norm_silu_post",
|
||||
"can_use_wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton",
|
||||
"wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton",
|
||||
"can_use_qk_rmsnorm_native": "norm.zimage_qk_rmsnorm_triton",
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""FP32 Wan normalization post-ops with the native L2 denominator supplied."""
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
|
||||
@cache_once
|
||||
def _module(x_dtype, affine_dtype, channels_last):
|
||||
args = make_cpp_args(x_dtype, affine_dtype, channels_last)
|
||||
return load_jit(
|
||||
"wan_norm_silu_post",
|
||||
*args,
|
||||
cuda_files=["elementwise/wan_norm_silu_post.cuh"],
|
||||
extra_cuda_cflags=["--fmad=false"],
|
||||
cuda_wrappers=[("run", f"wan_norm_silu_post<{args}>")],
|
||||
)
|
||||
|
||||
|
||||
def can_use_wan_norm_silu_post(x, denominator, gamma, bias=None):
|
||||
if (
|
||||
not x.is_cuda
|
||||
or torch.version.hip is not None
|
||||
or torch.is_grad_enabled()
|
||||
or x.requires_grad
|
||||
or x.ndim != 5
|
||||
or x.numel() == 0
|
||||
or x.dtype not in (torch.bfloat16, torch.float32)
|
||||
or x.shape[1] % 4
|
||||
or (x.shape[2] * x.shape[3] * x.shape[4]) % 4
|
||||
or not (
|
||||
x.is_contiguous() or x.is_contiguous(memory_format=torch.channels_last_3d)
|
||||
)
|
||||
or x.data_ptr() % 16
|
||||
or denominator.dtype != torch.float32
|
||||
or denominator.device != x.device
|
||||
or denominator.shape != (x.shape[0], 1, *x.shape[2:])
|
||||
or not denominator.is_contiguous()
|
||||
or denominator.data_ptr() % 16
|
||||
or gamma.dtype not in (torch.bfloat16, torch.float32)
|
||||
or gamma.device != x.device
|
||||
or gamma.numel() != x.shape[1]
|
||||
or not gamma.is_contiguous()
|
||||
or gamma.data_ptr() % 16
|
||||
):
|
||||
return False
|
||||
return bias is None or (
|
||||
bias.device == x.device
|
||||
and bias.dtype == gamma.dtype
|
||||
and bias.numel() == x.shape[1]
|
||||
and bias.is_contiguous()
|
||||
and bias.data_ptr() % 16 == 0
|
||||
)
|
||||
|
||||
|
||||
def _storage(x, channels_last):
|
||||
return x.permute(0, 2, 3, 4, 1).view(-1) if channels_last else x.view(-1)
|
||||
|
||||
|
||||
@register_custom_op(mutates_args=["out"])
|
||||
def _wan_norm_silu_post(
|
||||
x: torch.Tensor,
|
||||
denominator: torch.Tensor,
|
||||
gamma: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
scale: float,
|
||||
has_bias: bool,
|
||||
) -> None:
|
||||
channels_last = not x.is_contiguous()
|
||||
_module(x.dtype, gamma.dtype, channels_last).run(
|
||||
_storage(x, channels_last),
|
||||
denominator.view(-1),
|
||||
gamma.view(-1),
|
||||
bias.view(-1),
|
||||
_storage(out, channels_last),
|
||||
x.shape[1],
|
||||
x.shape[2] * x.shape[3] * x.shape[4],
|
||||
scale,
|
||||
has_bias,
|
||||
)
|
||||
|
||||
|
||||
def wan_norm_silu_post(x, denominator, gamma, bias=None, *, scale=None):
|
||||
if not can_use_wan_norm_silu_post(x, denominator, gamma, bias):
|
||||
raise ValueError("unsupported Wan normalization post-op inputs")
|
||||
out = torch.empty_like(x, dtype=torch.float32)
|
||||
_wan_norm_silu_post(
|
||||
x,
|
||||
denominator,
|
||||
gamma,
|
||||
gamma if bias is None else bias,
|
||||
out,
|
||||
float(x.shape[1] ** 0.5 if scale is None else scale),
|
||||
bias is not None,
|
||||
)
|
||||
return out
|
||||
@@ -6,8 +6,9 @@ Fuses every decoder ``WanRMS_norm -> SiLU`` chain into one Triton kernel on
|
||||
the channels_last_3d layout. Wrappers are installed once at VAE load and
|
||||
dispatch on a decode-scoped :class:`VaeFastPathGate`: ``quality="extra-high"``
|
||||
and ``quality="high"`` run the fused kernel (not bitwise-identical to aten,
|
||||
hence gated), while the ``"lossless"`` default runs the original module path
|
||||
bit-for-bit. Install is all-or-nothing and fail-closed.
|
||||
hence gated). The ``"lossless"`` path preserves aten's normalization reduction
|
||||
and fuses its FP32 post-ops after first-sight exactness verification.
|
||||
Install is all-or-nothing and fail-closed.
|
||||
"""
|
||||
|
||||
import torch
|
||||
@@ -24,9 +25,12 @@ logger = init_logger(__name__)
|
||||
|
||||
try:
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_use_nearest_upsample_nhwc,
|
||||
can_use_wan_norm_silu_post,
|
||||
can_use_wan_rmsnorm_silu,
|
||||
nearest_upsample_nhwc,
|
||||
wan_norm_silu_post,
|
||||
wan_rmsnorm_silu,
|
||||
)
|
||||
|
||||
@@ -36,10 +40,10 @@ except ImportError: # pragma: no cover
|
||||
|
||||
|
||||
class FusedWanRMSNormSiLU(nn.Module):
|
||||
"""``WanRMS_norm`` + SiLU fused via the channels_last_3d Triton kernel;
|
||||
falls back to the original op chain (bit-identical to norm + ``nn.SiLU``)
|
||||
for unsupported inputs and whenever the gate is off. Steps aside under
|
||||
``torch.compile``, where Inductor already fuses this chain."""
|
||||
"""Quality-gated full fusion or lossless FP32 post-op fusion.
|
||||
|
||||
Unsupported inputs and ``torch.compile`` use the original op chain.
|
||||
"""
|
||||
|
||||
def __init__(self, norm: nn.Module, gate: VaeFastPathGate) -> None:
|
||||
super().__init__()
|
||||
@@ -50,12 +54,58 @@ class FusedWanRMSNormSiLU(nn.Module):
|
||||
self.bias = norm.bias
|
||||
self.scale = float(norm.scale)
|
||||
self._sgl_gate = gate
|
||||
self._post_gate = BitExactFusionGate(
|
||||
"Wan norm+SiLU post-ops", per_signature=True
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self._sgl_gate.enabled and not torch.compiler.is_compiling():
|
||||
bias = self.bias if isinstance(self.bias, torch.Tensor) else None
|
||||
if can_use_wan_rmsnorm_silu(x, self.gamma, bias):
|
||||
return wan_rmsnorm_silu(x, self.gamma, bias, rms_scale=self.scale)
|
||||
if (
|
||||
not torch.compiler.is_compiling()
|
||||
and not self._post_gate.disabled
|
||||
and x.is_cuda
|
||||
and torch.version.hip is None
|
||||
and not torch.is_grad_enabled()
|
||||
and x.ndim == 5
|
||||
and (
|
||||
x.dtype == torch.float32
|
||||
or (x.dtype == torch.bfloat16 and torch.is_autocast_enabled("cuda"))
|
||||
)
|
||||
and (isinstance(self.bias, torch.Tensor) or self.bias == 0.0)
|
||||
):
|
||||
sig = (x.device, x.dtype, x.shape, x.stride(), self.gamma.dtype)
|
||||
verified = self._post_gate.is_verified(sig)
|
||||
if verified or not torch.cuda.is_current_stream_capturing():
|
||||
# Keep this outside the custom op: CUDA autocast promotes the
|
||||
# native norm to FP32, including for BF16 VAE activations.
|
||||
denominator = x.norm(p=2, dim=1, keepdim=True).clamp_min(1e-12)
|
||||
bias = self.bias if isinstance(self.bias, torch.Tensor) else None
|
||||
if can_use_wan_norm_silu_post(x, denominator, self.gamma, bias):
|
||||
try:
|
||||
out = wan_norm_silu_post(
|
||||
x, denominator, self.gamma, bias, scale=self.scale
|
||||
)
|
||||
if verified:
|
||||
return out
|
||||
ref = F.silu(
|
||||
(x / denominator) * self.scale * self.gamma + self.bias
|
||||
)
|
||||
return self._post_gate.accept_or_fallback(
|
||||
out,
|
||||
ref,
|
||||
sig=sig,
|
||||
equal=lambda a, b: torch.equal(
|
||||
a.view(torch.int32), b.view(torch.int32)
|
||||
),
|
||||
logger=logger,
|
||||
)
|
||||
except Exception as exc:
|
||||
self._post_gate.on_exception(exc, logger=logger)
|
||||
# Unsupported layouts/dtypes still reuse the native reduction.
|
||||
return F.silu((x / denominator) * self.scale * self.gamma + self.bias)
|
||||
# WanRMS_norm.forward (channel-first) + SiLU, same ops in the same
|
||||
# order, so the off-path stays bit-identical.
|
||||
return F.silu(F.normalize(x, dim=1) * self.scale * self.gamma + self.bias)
|
||||
|
||||
Reference in New Issue
Block a user