[Diffusion] Fuse lossless Wan VAE post-ops for LongLive 2 I2V (#40405)
This commit is contained in:
@@ -112,6 +112,7 @@ These fusion families mount under both `quality="extra-high"` and
|
||||
| `apply_group_norm_silu` | Triton | close | `GroupNorm + SiLU`, NCHW-contiguous |
|
||||
| `group_norm_silu_4d`, `group_norm_silu_rows` | Triton | close | channels-last GroupNorm(+SiLU); what lets a VAE decoder run channels_last end to end with no `nchwToNhwc` transposes |
|
||||
| `wan_rmsnorm_silu` | Triton | close | Wan VAE `channels_last_3d` RMSNorm + SiLU |
|
||||
| `wan_norm_silu_post` | JIT CUDA | bit-exact, first-sight verified | Wan VAE FP32 divide, scale, affine, and SiLU after the unchanged native L2 reduction; dense NCDHW and `channels_last_3d` |
|
||||
|
||||
### adaLN modulation and gating
|
||||
|
||||
@@ -224,7 +225,7 @@ Kernels are written against a specific eager chain in a specific model, so cover
|
||||
| Sana | LN+modulate, GLUMB bias+SiLU / bias+GLU, residual-gate add |
|
||||
| SANA-Video | Packed QKV/KV; paired fp64 interleaved RoPE; LN+modulate, GLUMB bias+SiLU / bias+GLU, and residual-gate add during BCG; BF16-input linear attention at `quality=extra-high` or `quality=high` |
|
||||
| Sana-WM | bidirectional gated delta-net, fused QK inverse-RMS |
|
||||
| Wan | temb table slices; VAE cat+pad and DupUp3D add, `channels_last_3d` RMSNorm+SiLU |
|
||||
| Wan / LongLive 2 VAE | temb table slices (Wan); VAE cat+pad and DupUp3D add; lossless FP32 normalization post-ops; full `channels_last_3d` RMSNorm+SiLU at `quality=extra-high` or `quality=high` |
|
||||
| Cosmos3 / Krea2 / MiniMax-H3 | QK-norm + RoPE (Krea2 also CuTe-DSL norm+scale/shift; MiniMax-H3 also indexed modulation) |
|
||||
| FLUX.2 VAE / HunyuanVAE / latent upsampler | GroupNorm + SiLU (channels-last two-pass for FLUX.2) |
|
||||
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
#include <sgl_kernel/vec.cuh>
|
||||
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
/// \brief Fuse FP32 divide, scale, affine, and SiLU without changing aten's L2 reduction.
|
||||
template <typename X, typename A, bool kChannelsLast>
|
||||
__global__ void wan_norm_silu_post_kernel(
|
||||
const X* __restrict__ input,
|
||||
const float* __restrict__ denominator,
|
||||
const A* __restrict__ gamma,
|
||||
const A* __restrict__ bias,
|
||||
float* __restrict__ output,
|
||||
int64_t num_vecs,
|
||||
int64_t channels,
|
||||
int64_t spatial,
|
||||
float scale,
|
||||
bool has_bias) {
|
||||
constexpr int kVec = 4;
|
||||
const int64_t step = static_cast<int64_t>(blockDim.x) * gridDim.x;
|
||||
for (int64_t vec = static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x; vec < num_vecs; vec += step) {
|
||||
device::AlignedVector<X, kVec> x;
|
||||
device::AlignedVector<float, kVec> d;
|
||||
device::AlignedVector<A, kVec> g;
|
||||
device::AlignedVector<A, kVec> b;
|
||||
device::AlignedVector<float, kVec> result;
|
||||
x.load(input, vec);
|
||||
float shared_d = 0.f, shared_g = 0.f, shared_b = 0.f;
|
||||
if constexpr (kChannelsLast) {
|
||||
const int64_t channel = vec * kVec % channels;
|
||||
shared_d = denominator[vec * kVec / channels];
|
||||
g.load(gamma, channel / kVec);
|
||||
if (has_bias) b.load(bias, channel / kVec);
|
||||
} else {
|
||||
const int64_t channel = vec * kVec / spatial % channels;
|
||||
const int64_t batch = vec * kVec / (spatial * channels);
|
||||
const int64_t pixel = vec * kVec % spatial;
|
||||
d.load(denominator, (batch * spatial + pixel) / kVec);
|
||||
shared_g = device::cast<float>(gamma[channel]);
|
||||
if (has_bias) shared_b = device::cast<float>(bias[channel]);
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kVec; ++i) {
|
||||
const float denom = kChannelsLast ? shared_d : d[i];
|
||||
const float weight = kChannelsLast ? device::cast<float>(g[i]) : shared_g;
|
||||
const float offset = has_bias ? (kChannelsLast ? device::cast<float>(b[i]) : shared_b) : 0.f;
|
||||
float y = __fdiv_rn(device::cast<float>(x[i]), denom);
|
||||
y = __fmul_rn(y, scale);
|
||||
y = __fmul_rn(y, weight);
|
||||
y = __fadd_rn(y, offset);
|
||||
result[i] = __fdiv_rn(y, __fadd_rn(1.f, expf(-y)));
|
||||
}
|
||||
result.store(output, vec);
|
||||
}
|
||||
}
|
||||
|
||||
/// \brief Launch normalization post-ops for dense NCDHW or channels-last storage.
|
||||
/// \param denominator Native FP32 L2 norm, clamped to the normalization epsilon.
|
||||
template <typename X, typename A, bool kChannelsLast>
|
||||
void wan_norm_silu_post(
|
||||
tvm::ffi::TensorView input,
|
||||
tvm::ffi::TensorView denominator,
|
||||
tvm::ffi::TensorView gamma,
|
||||
tvm::ffi::TensorView bias,
|
||||
tvm::ffi::TensorView output,
|
||||
int64_t channels,
|
||||
int64_t spatial,
|
||||
double scale,
|
||||
bool has_bias) {
|
||||
using namespace host;
|
||||
auto elements = SymbolicSize{"elements"};
|
||||
auto pixels = SymbolicSize{"pixels"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLCUDA>();
|
||||
TensorMatcher({elements}).with_dtype<X>().with_device(device_).verify(input);
|
||||
TensorMatcher({pixels}).with_dtype<float>().with_device(device_).verify(denominator);
|
||||
TensorMatcher({channels}).with_dtype<A>().with_device(device_).verify(gamma);
|
||||
TensorMatcher({channels}).with_dtype<A>().with_device(device_).verify(bias);
|
||||
TensorMatcher({elements}).with_dtype<float>().with_device(device_).verify(output);
|
||||
const int64_t count = elements.unwrap();
|
||||
CHECK_HOST(channels > 0 && channels % 4 == 0 && spatial > 0 && spatial % 4 == 0);
|
||||
CHECK_HOST(count > 0 && count % (channels * spatial) == 0);
|
||||
CHECK_HOST(pixels.unwrap() == count / channels);
|
||||
constexpr int64_t threads = 256;
|
||||
const auto kernel = wan_norm_silu_post_kernel<X, A, kChannelsLast>;
|
||||
const auto blocks_per_sm = runtime::get_blocks_per_sm(kernel, threads);
|
||||
const auto sms = runtime::get_sm_count(device_.unwrap().device_id);
|
||||
const auto blocks = std::min<int64_t>(sms * blocks_per_sm, div_ceil(count / 4, threads));
|
||||
LaunchKernel(blocks, threads, device_.unwrap())(
|
||||
kernel,
|
||||
static_cast<const X*>(input.data_ptr()),
|
||||
static_cast<const float*>(denominator.data_ptr()),
|
||||
static_cast<const A*>(gamma.data_ptr()),
|
||||
static_cast<const A*>(bias.data_ptr()),
|
||||
static_cast<float*>(output.data_ptr()),
|
||||
count / 4,
|
||||
channels,
|
||||
spatial,
|
||||
static_cast<float>(scale),
|
||||
has_bias);
|
||||
}
|
||||
|
||||
} // namespace sglang
|
||||
@@ -84,6 +84,13 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
|
||||
_CUDA,
|
||||
"Wan VAE channels_last_3d RMSNorm + SiLU.",
|
||||
),
|
||||
(
|
||||
"diffusion.wan_norm_silu_post",
|
||||
KernelBackend.JIT,
|
||||
"norm.wan_norm_silu_post:wan_norm_silu_post",
|
||||
_CUDA,
|
||||
"Wan VAE FP32 normalization post-ops with the native denominator.",
|
||||
),
|
||||
(
|
||||
"diffusion.rmsnorm_scale_shift",
|
||||
KernelBackend.TRITON,
|
||||
@@ -580,6 +587,8 @@ _EXPORTS: dict[str, str] = {
|
||||
"try_fused_scale_residual_norm_scale_shift_fp8": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
|
||||
"validate_scale_shift": "norm.scale_residual_norm_cutedsl",
|
||||
"try_fused_scale_residual_norm_scale_shift_nvfp4": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
|
||||
"can_use_wan_norm_silu_post": "norm.wan_norm_silu_post",
|
||||
"wan_norm_silu_post": "norm.wan_norm_silu_post",
|
||||
"can_use_wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton",
|
||||
"wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton",
|
||||
"can_use_qk_rmsnorm_native": "norm.zimage_qk_rmsnorm_triton",
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""FP32 Wan normalization post-ops with the native L2 denominator supplied."""
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
|
||||
@cache_once
|
||||
def _module(x_dtype, affine_dtype, channels_last):
|
||||
args = make_cpp_args(x_dtype, affine_dtype, channels_last)
|
||||
return load_jit(
|
||||
"wan_norm_silu_post",
|
||||
*args,
|
||||
cuda_files=["elementwise/wan_norm_silu_post.cuh"],
|
||||
extra_cuda_cflags=["--fmad=false"],
|
||||
cuda_wrappers=[("run", f"wan_norm_silu_post<{args}>")],
|
||||
)
|
||||
|
||||
|
||||
def can_use_wan_norm_silu_post(x, denominator, gamma, bias=None):
|
||||
if (
|
||||
not x.is_cuda
|
||||
or torch.version.hip is not None
|
||||
or torch.is_grad_enabled()
|
||||
or x.requires_grad
|
||||
or x.ndim != 5
|
||||
or x.numel() == 0
|
||||
or x.dtype not in (torch.bfloat16, torch.float32)
|
||||
or x.shape[1] % 4
|
||||
or (x.shape[2] * x.shape[3] * x.shape[4]) % 4
|
||||
or not (
|
||||
x.is_contiguous() or x.is_contiguous(memory_format=torch.channels_last_3d)
|
||||
)
|
||||
or x.data_ptr() % 16
|
||||
or denominator.dtype != torch.float32
|
||||
or denominator.device != x.device
|
||||
or denominator.shape != (x.shape[0], 1, *x.shape[2:])
|
||||
or not denominator.is_contiguous()
|
||||
or denominator.data_ptr() % 16
|
||||
or gamma.dtype not in (torch.bfloat16, torch.float32)
|
||||
or gamma.device != x.device
|
||||
or gamma.numel() != x.shape[1]
|
||||
or not gamma.is_contiguous()
|
||||
or gamma.data_ptr() % 16
|
||||
):
|
||||
return False
|
||||
return bias is None or (
|
||||
bias.device == x.device
|
||||
and bias.dtype == gamma.dtype
|
||||
and bias.numel() == x.shape[1]
|
||||
and bias.is_contiguous()
|
||||
and bias.data_ptr() % 16 == 0
|
||||
)
|
||||
|
||||
|
||||
def _storage(x, channels_last):
|
||||
return x.permute(0, 2, 3, 4, 1).view(-1) if channels_last else x.view(-1)
|
||||
|
||||
|
||||
@register_custom_op(mutates_args=["out"])
|
||||
def _wan_norm_silu_post(
|
||||
x: torch.Tensor,
|
||||
denominator: torch.Tensor,
|
||||
gamma: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
scale: float,
|
||||
has_bias: bool,
|
||||
) -> None:
|
||||
channels_last = not x.is_contiguous()
|
||||
_module(x.dtype, gamma.dtype, channels_last).run(
|
||||
_storage(x, channels_last),
|
||||
denominator.view(-1),
|
||||
gamma.view(-1),
|
||||
bias.view(-1),
|
||||
_storage(out, channels_last),
|
||||
x.shape[1],
|
||||
x.shape[2] * x.shape[3] * x.shape[4],
|
||||
scale,
|
||||
has_bias,
|
||||
)
|
||||
|
||||
|
||||
def wan_norm_silu_post(x, denominator, gamma, bias=None, *, scale=None):
|
||||
if not can_use_wan_norm_silu_post(x, denominator, gamma, bias):
|
||||
raise ValueError("unsupported Wan normalization post-op inputs")
|
||||
out = torch.empty_like(x, dtype=torch.float32)
|
||||
_wan_norm_silu_post(
|
||||
x,
|
||||
denominator,
|
||||
gamma,
|
||||
gamma if bias is None else bias,
|
||||
out,
|
||||
float(x.shape[1] ** 0.5 if scale is None else scale),
|
||||
bias is not None,
|
||||
)
|
||||
return out
|
||||
@@ -6,8 +6,9 @@ Fuses every decoder ``WanRMS_norm -> SiLU`` chain into one Triton kernel on
|
||||
the channels_last_3d layout. Wrappers are installed once at VAE load and
|
||||
dispatch on a decode-scoped :class:`VaeFastPathGate`: ``quality="extra-high"``
|
||||
and ``quality="high"`` run the fused kernel (not bitwise-identical to aten,
|
||||
hence gated), while the ``"lossless"`` default runs the original module path
|
||||
bit-for-bit. Install is all-or-nothing and fail-closed.
|
||||
hence gated). The ``"lossless"`` path preserves aten's normalization reduction
|
||||
and fuses its FP32 post-ops after first-sight exactness verification.
|
||||
Install is all-or-nothing and fail-closed.
|
||||
"""
|
||||
|
||||
import torch
|
||||
@@ -24,9 +25,12 @@ logger = init_logger(__name__)
|
||||
|
||||
try:
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_use_nearest_upsample_nhwc,
|
||||
can_use_wan_norm_silu_post,
|
||||
can_use_wan_rmsnorm_silu,
|
||||
nearest_upsample_nhwc,
|
||||
wan_norm_silu_post,
|
||||
wan_rmsnorm_silu,
|
||||
)
|
||||
|
||||
@@ -36,10 +40,10 @@ except ImportError: # pragma: no cover
|
||||
|
||||
|
||||
class FusedWanRMSNormSiLU(nn.Module):
|
||||
"""``WanRMS_norm`` + SiLU fused via the channels_last_3d Triton kernel;
|
||||
falls back to the original op chain (bit-identical to norm + ``nn.SiLU``)
|
||||
for unsupported inputs and whenever the gate is off. Steps aside under
|
||||
``torch.compile``, where Inductor already fuses this chain."""
|
||||
"""Quality-gated full fusion or lossless FP32 post-op fusion.
|
||||
|
||||
Unsupported inputs and ``torch.compile`` use the original op chain.
|
||||
"""
|
||||
|
||||
def __init__(self, norm: nn.Module, gate: VaeFastPathGate) -> None:
|
||||
super().__init__()
|
||||
@@ -50,12 +54,58 @@ class FusedWanRMSNormSiLU(nn.Module):
|
||||
self.bias = norm.bias
|
||||
self.scale = float(norm.scale)
|
||||
self._sgl_gate = gate
|
||||
self._post_gate = BitExactFusionGate(
|
||||
"Wan norm+SiLU post-ops", per_signature=True
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self._sgl_gate.enabled and not torch.compiler.is_compiling():
|
||||
bias = self.bias if isinstance(self.bias, torch.Tensor) else None
|
||||
if can_use_wan_rmsnorm_silu(x, self.gamma, bias):
|
||||
return wan_rmsnorm_silu(x, self.gamma, bias, rms_scale=self.scale)
|
||||
if (
|
||||
not torch.compiler.is_compiling()
|
||||
and not self._post_gate.disabled
|
||||
and x.is_cuda
|
||||
and torch.version.hip is None
|
||||
and not torch.is_grad_enabled()
|
||||
and x.ndim == 5
|
||||
and (
|
||||
x.dtype == torch.float32
|
||||
or (x.dtype == torch.bfloat16 and torch.is_autocast_enabled("cuda"))
|
||||
)
|
||||
and (isinstance(self.bias, torch.Tensor) or self.bias == 0.0)
|
||||
):
|
||||
sig = (x.device, x.dtype, x.shape, x.stride(), self.gamma.dtype)
|
||||
verified = self._post_gate.is_verified(sig)
|
||||
if verified or not torch.cuda.is_current_stream_capturing():
|
||||
# Keep this outside the custom op: CUDA autocast promotes the
|
||||
# native norm to FP32, including for BF16 VAE activations.
|
||||
denominator = x.norm(p=2, dim=1, keepdim=True).clamp_min(1e-12)
|
||||
bias = self.bias if isinstance(self.bias, torch.Tensor) else None
|
||||
if can_use_wan_norm_silu_post(x, denominator, self.gamma, bias):
|
||||
try:
|
||||
out = wan_norm_silu_post(
|
||||
x, denominator, self.gamma, bias, scale=self.scale
|
||||
)
|
||||
if verified:
|
||||
return out
|
||||
ref = F.silu(
|
||||
(x / denominator) * self.scale * self.gamma + self.bias
|
||||
)
|
||||
return self._post_gate.accept_or_fallback(
|
||||
out,
|
||||
ref,
|
||||
sig=sig,
|
||||
equal=lambda a, b: torch.equal(
|
||||
a.view(torch.int32), b.view(torch.int32)
|
||||
),
|
||||
logger=logger,
|
||||
)
|
||||
except Exception as exc:
|
||||
self._post_gate.on_exception(exc, logger=logger)
|
||||
# Unsupported layouts/dtypes still reuse the native reduction.
|
||||
return F.silu((x / denominator) * self.scale * self.gamma + self.bias)
|
||||
# WanRMS_norm.forward (channel-first) + SiLU, same ops in the same
|
||||
# order, so the off-path stays bit-identical.
|
||||
return F.silu(F.normalize(x, dim=1) * self.scale * self.gamma + self.bias)
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Lossless Wan normalization post-ops, dtype boundaries, and CUDA replay."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
import sglang.multimodal_gen.runtime.models.vaes.wan_vae_cuda_opt as wan
|
||||
from sglang.kernels.ops.diffusion import wan_norm_silu_post
|
||||
from sglang.multimodal_gen.runtime.models.vaes.fast_path_gate import VaeFastPathGate
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=45, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
def reference(x, gamma, bias, scale):
|
||||
return F.silu(F.normalize(x, dim=1) * scale * gamma + bias)
|
||||
|
||||
|
||||
class TestWanNormSiLUPost(CustomTestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
if not torch.cuda.is_available() or torch.version.hip is not None:
|
||||
self.skipTest("NVIDIA CUDA required")
|
||||
torch.manual_seed(1729)
|
||||
|
||||
def assertBitsEqual(self, actual, expected):
|
||||
self.assertEqual(actual.dtype, expected.dtype)
|
||||
self.assertEqual(actual.stride(), expected.stride())
|
||||
self.assertTrue(
|
||||
torch.equal(actual.view(torch.int32), expected.view(torch.int32))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def module(channels=64, *, bias=False, dtype=torch.bfloat16):
|
||||
gamma = torch.nn.Parameter(
|
||||
torch.randn(channels, 1, 1, 1, device="cuda", dtype=dtype)
|
||||
)
|
||||
offset = torch.nn.Parameter(torch.randn_like(gamma)) if bias else 0.0
|
||||
norm = SimpleNamespace(gamma=gamma, bias=offset, scale=channels**0.5)
|
||||
return wan.FusedWanRMSNormSiLU(norm, VaeFastPathGate())
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_autocast_promotion_layout_and_affine(self):
|
||||
for dtype in (torch.bfloat16, torch.float32):
|
||||
for affine in (torch.bfloat16, torch.float32):
|
||||
for layout in (torch.contiguous_format, torch.channels_last_3d):
|
||||
for bias in (False, True):
|
||||
with self.subTest(
|
||||
dtype=dtype, affine=affine, layout=layout, bias=bias
|
||||
):
|
||||
module = self.module(bias=bias, dtype=affine)
|
||||
x = torch.randn(
|
||||
2, 64, 3, 8, 12, device="cuda", dtype=dtype
|
||||
).to(memory_format=layout)
|
||||
original = x.clone()
|
||||
with torch.autocast("cuda", dtype=torch.bfloat16):
|
||||
expected = reference(
|
||||
x, module.gamma, module.bias, module.scale
|
||||
)
|
||||
actual = module(x)
|
||||
self.assertBitsEqual(actual, expected)
|
||||
self.assertTrue(module._post_gate.verified)
|
||||
self.assertFalse(module._post_gate.disabled)
|
||||
x.add_(0.125)
|
||||
self.assertBitsEqual(
|
||||
module(x),
|
||||
reference(
|
||||
x, module.gamma, module.bias, module.scale
|
||||
),
|
||||
)
|
||||
self.assertTrue(torch.equal(original + 0.125, x))
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_all_finite_bf16_and_fp32_silu_boundaries(self):
|
||||
bits = torch.arange(65536, device="cuda", dtype=torch.int32).to(torch.int16)
|
||||
values = bits.view(torch.bfloat16)
|
||||
finite = values[torch.isfinite(values)].reshape(1, 64, 1, 1, 1020)
|
||||
fp32 = (
|
||||
torch.tensor(
|
||||
[
|
||||
-110.0,
|
||||
-104.0,
|
||||
-100.0,
|
||||
-90.0,
|
||||
-88.0,
|
||||
-80.0,
|
||||
-20.0,
|
||||
-1.0,
|
||||
-0.0,
|
||||
0.0,
|
||||
1e-40,
|
||||
-1e-40,
|
||||
1.0,
|
||||
20.0,
|
||||
80.0,
|
||||
100.0,
|
||||
],
|
||||
device="cuda",
|
||||
)
|
||||
.repeat(64)
|
||||
.reshape(1, 64, 1, 1, 16)
|
||||
)
|
||||
for domain in (finite, fp32):
|
||||
for layout in (torch.contiguous_format, torch.channels_last_3d):
|
||||
x = domain.to(memory_format=layout)
|
||||
denominator = torch.ones((1, 1, *x.shape[2:]), device="cuda")
|
||||
gamma = torch.ones((64, 1, 1, 1), device="cuda", dtype=torch.bfloat16)
|
||||
expected = F.silu((x / denominator) * 1.0 * gamma + 0.0)
|
||||
self.assertBitsEqual(
|
||||
wan_norm_silu_post(x, denominator, gamma, scale=1.0), expected
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_graph_replay_recomputes_norm_and_affine(self):
|
||||
for layout in (torch.contiguous_format, torch.channels_last_3d):
|
||||
module = self.module(bias=True)
|
||||
x = torch.randn(2, 64, 3, 8, 12, device="cuda", dtype=torch.bfloat16).to(
|
||||
memory_format=layout
|
||||
)
|
||||
with torch.autocast("cuda", dtype=torch.bfloat16):
|
||||
module(x)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual = module(x)
|
||||
x.mul_(0.5)
|
||||
module.gamma.add_(0.25)
|
||||
module.bias.sub_(0.125)
|
||||
graph.replay()
|
||||
self.assertBitsEqual(
|
||||
actual, reference(x, module.gamma, module.bias, module.scale)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user